{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AI第三周作业——XGBoost "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.模型训练：超参数调优"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>...</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "      <th>interest_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.5</td>\n",
       "      <td>3</td>\n",
       "      <td>3000</td>\n",
       "      <td>1200.0</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>-1.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>5465</td>\n",
       "      <td>2732.5</td>\n",
       "      <td>1821.666667</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2850</td>\n",
       "      <td>1425.0</td>\n",
       "      <td>1425.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3275</td>\n",
       "      <td>1637.5</td>\n",
       "      <td>1637.500000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>18</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>4</td>\n",
       "      <td>3350</td>\n",
       "      <td>1675.0</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>28</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 228 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathrooms  bedrooms  price  price_bathrooms  price_bedrooms  room_diff  \\\n",
       "0        1.5         3   3000           1200.0      750.000000       -1.5   \n",
       "1        1.0         2   5465           2732.5     1821.666667       -1.0   \n",
       "2        1.0         1   2850           1425.0     1425.000000        0.0   \n",
       "3        1.0         1   3275           1637.5     1637.500000        0.0   \n",
       "4        1.0         4   3350           1675.0      670.000000       -3.0   \n",
       "\n",
       "   room_num  Year  Month  Day       ...        walk  walls  war  washer  \\\n",
       "0       4.5  2016      6   24       ...           0      0    0       0   \n",
       "1       3.0  2016      6   12       ...           0      0    0       0   \n",
       "2       2.0  2016      4   17       ...           0      0    0       0   \n",
       "3       2.0  2016      4   18       ...           0      0    0       0   \n",
       "4       5.0  2016      4   28       ...           0      0    1       0   \n",
       "\n",
       "   water  wheelchair  wifi  windows  work  interest_level  \n",
       "0      0           0     0        0     0               1  \n",
       "1      0           0     0        0     0               2  \n",
       "2      0           0     0        0     0               0  \n",
       "3      0           0     0        0     0               2  \n",
       "4      0           0     0        0     0               2  \n",
       "\n",
       "[5 rows x 228 columns]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dpath = './data/'\n",
    "pd.set_option('display.max_info_columns', 500)\n",
    "train = pd.read_csv(dpath + \"RentListingInquries_FE_train.csv\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 49352 entries, 0 to 49351\n",
      "Data columns (total 228 columns):\n",
      "bathrooms                 49352 non-null float64\n",
      "bedrooms                  49352 non-null int64\n",
      "price                     49352 non-null int64\n",
      "price_bathrooms           49352 non-null float64\n",
      "price_bedrooms            49352 non-null float64\n",
      "room_diff                 49352 non-null float64\n",
      "room_num                  49352 non-null float64\n",
      "Year                      49352 non-null int64\n",
      "Month                     49352 non-null int64\n",
      "Day                       49352 non-null int64\n",
      "Wday                      49352 non-null int64\n",
      "Yday                      49352 non-null int64\n",
      "hour                      49352 non-null int64\n",
      "top_10_manager            49352 non-null int64\n",
      "top_25_manager            49352 non-null int64\n",
      "top_5_manager             49352 non-null int64\n",
      "top_50_manager            49352 non-null int64\n",
      "top_1_manager             49352 non-null int64\n",
      "top_2_manager             49352 non-null int64\n",
      "top_15_manager            49352 non-null int64\n",
      "top_20_manager            49352 non-null int64\n",
      "top_30_manager            49352 non-null int64\n",
      "cenroid                   49352 non-null int64\n",
      "distance                  49352 non-null float64\n",
      "display_address_pred_0    49352 non-null float64\n",
      "display_address_pred_1    49352 non-null float64\n",
      "display_address_pred_2    49352 non-null float64\n",
      "1br                       49352 non-null int64\n",
      "24                        49352 non-null int64\n",
      "2br                       49352 non-null int64\n",
      "3br                       49352 non-null int64\n",
      "ac                        49352 non-null int64\n",
      "access                    49352 non-null int64\n",
      "actual                    49352 non-null int64\n",
      "air                       49352 non-null int64\n",
      "allowed                   49352 non-null int64\n",
      "appliances                49352 non-null int64\n",
      "approval                  49352 non-null int64\n",
      "apt                       49352 non-null int64\n",
      "areas                     49352 non-null int64\n",
      "attended                  49352 non-null int64\n",
      "available                 49352 non-null int64\n",
      "backyard                  49352 non-null int64\n",
      "balcony                   49352 non-null int64\n",
      "basement                  49352 non-null int64\n",
      "bath                      49352 non-null int64\n",
      "bathroom                  49352 non-null int64\n",
      "bedford                   49352 non-null int64\n",
      "bedroom                   49352 non-null int64\n",
      "bedrooms.1                49352 non-null int64\n",
      "bike                      49352 non-null int64\n",
      "billiards                 49352 non-null int64\n",
      "bldg                      49352 non-null int64\n",
      "blks                      49352 non-null int64\n",
      "brand                     49352 non-null int64\n",
      "brick                     49352 non-null int64\n",
      "brownstone                49352 non-null int64\n",
      "building                  49352 non-null int64\n",
      "business                  49352 non-null int64\n",
      "cable                     49352 non-null int64\n",
      "cats                      49352 non-null int64\n",
      "ceiling                   49352 non-null int64\n",
      "ceilings                  49352 non-null int64\n",
      "center                    49352 non-null int64\n",
      "central                   49352 non-null int64\n",
      "chef                      49352 non-null int64\n",
      "children                  49352 non-null int64\n",
      "childrens                 49352 non-null int64\n",
      "clean                     49352 non-null int64\n",
      "cleaning                  49352 non-null int64\n",
      "close                     49352 non-null int64\n",
      "closet                    49352 non-null int64\n",
      "closets                   49352 non-null int64\n",
      "club                      49352 non-null int64\n",
      "common                    49352 non-null int64\n",
      "concierge                 49352 non-null int64\n",
      "conditioning              49352 non-null int64\n",
      "construction              49352 non-null int64\n",
      "courtyard                 49352 non-null int64\n",
      "deck                      49352 non-null int64\n",
      "decorative                49352 non-null int64\n",
      "details                   49352 non-null int64\n",
      "dining                    49352 non-null int64\n",
      "dishwasher                49352 non-null int64\n",
      "dogs                      49352 non-null int64\n",
      "doorman                   49352 non-null int64\n",
      "dry                       49352 non-null int64\n",
      "dryer                     49352 non-null int64\n",
      "duplex                    49352 non-null int64\n",
      "eat                       49352 non-null int64\n",
      "elev                      49352 non-null int64\n",
      "elevator                  49352 non-null int64\n",
      "entertainment             49352 non-null int64\n",
      "exclusive                 49352 non-null int64\n",
      "exposed                   49352 non-null int64\n",
      "facilities                49352 non-null int64\n",
      "fee                       49352 non-null int64\n",
      "fireplace                 49352 non-null int64\n",
      "fitness                   49352 non-null int64\n",
      "flex                      49352 non-null int64\n",
      "floor                     49352 non-null int64\n",
      "floors                    49352 non-null int64\n",
      "free                      49352 non-null int64\n",
      "friendly                  49352 non-null int64\n",
      "ft                        49352 non-null int64\n",
      "fully                     49352 non-null int64\n",
      "furnished                 49352 non-null int64\n",
      "garage                    49352 non-null int64\n",
      "garden                    49352 non-null int64\n",
      "granite                   49352 non-null int64\n",
      "green                     49352 non-null int64\n",
      "gut                       49352 non-null int64\n",
      "gym                       49352 non-null int64\n",
      "hardwood                  49352 non-null int64\n",
      "health                    49352 non-null int64\n",
      "hi                        49352 non-null int64\n",
      "high                      49352 non-null int64\n",
      "highrise                  49352 non-null int64\n",
      "home                      49352 non-null int64\n",
      "hour.1                    49352 non-null int64\n",
      "housekeeping              49352 non-null int64\n",
      "huge                      49352 non-null int64\n",
      "included                  49352 non-null int64\n",
      "indoor                    49352 non-null int64\n",
      "intercom                  49352 non-null int64\n",
      "internet                  49352 non-null int64\n",
      "kitchen                   49352 non-null int64\n",
      "large                     49352 non-null int64\n",
      "laundry                   49352 non-null int64\n",
      "level                     49352 non-null int64\n",
      "light                     49352 non-null int64\n",
      "live                      49352 non-null int64\n",
      "living                    49352 non-null int64\n",
      "lndry                     49352 non-null int64\n",
      "lobby                     49352 non-null int64\n",
      "loft                      49352 non-null int64\n",
      "lot                       49352 non-null int64\n",
      "lounge                    49352 non-null int64\n",
      "lowrise                   49352 non-null int64\n",
      "luxury                    49352 non-null int64\n",
      "marble                    49352 non-null int64\n",
      "massive                   49352 non-null int64\n",
      "microwave                 49352 non-null int64\n",
      "midrise                   49352 non-null int64\n",
      "modern                    49352 non-null int64\n",
      "month                     49352 non-null int64\n",
      "multi                     49352 non-null int64\n",
      "natural                   49352 non-null int64\n",
      "new                       49352 non-null int64\n",
      "newly                     49352 non-null int64\n",
      "ok                        49352 non-null int64\n",
      "outdoor                   49352 non-null int64\n",
      "oversized                 49352 non-null int64\n",
      "park                      49352 non-null int64\n",
      "parking                   49352 non-null int64\n",
      "patio                     49352 non-null int64\n",
      "pet                       49352 non-null int64\n",
      "pets                      49352 non-null int64\n",
      "photos                    49352 non-null int64\n",
      "playroom                  49352 non-null int64\n",
      "pool                      49352 non-null int64\n",
      "post                      49352 non-null int64\n",
      "pre                       49352 non-null int64\n",
      "prewar                    49352 non-null int64\n",
      "private                   49352 non-null int64\n",
      "publicoutdoor             49352 non-null int64\n",
      "queen                     49352 non-null int64\n",
      "ramp                      49352 non-null int64\n",
      "reduced                   49352 non-null int64\n",
      "renovated                 49352 non-null int64\n",
      "rent                      49352 non-null int64\n",
      "residents                 49352 non-null int64\n",
      "rise                      49352 non-null int64\n",
      "roof                      49352 non-null int64\n",
      "roofdeck                  49352 non-null int64\n",
      "rooftop                   49352 non-null int64\n",
      "room                      49352 non-null int64\n",
      "sauna                     49352 non-null int64\n",
      "service                   49352 non-null int64\n",
      "services                  49352 non-null int64\n",
      "share                     49352 non-null int64\n",
      "shared                    49352 non-null int64\n",
      "shares                    49352 non-null int64\n",
      "short                     49352 non-null int64\n",
      "simplex                   49352 non-null int64\n",
      "site                      49352 non-null int64\n",
      "sized                     49352 non-null int64\n",
      "skylight                  49352 non-null int64\n",
      "space                     49352 non-null int64\n",
      "spacious                  49352 non-null int64\n",
      "speed                     49352 non-null int64\n",
      "sprawling                 49352 non-null int64\n",
      "ss                        49352 non-null int64\n",
      "stainless                 49352 non-null int64\n",
      "steel                     49352 non-null int64\n",
      "steps                     49352 non-null int64\n",
      "storage                   49352 non-null int64\n",
      "studio                    49352 non-null int64\n",
      "sublet                    49352 non-null int64\n",
      "subway                    49352 non-null int64\n",
      "super                     49352 non-null int64\n",
      "superintendent            49352 non-null int64\n",
      "swimming                  49352 non-null int64\n",
      "tenant                    49352 non-null int64\n",
      "term                      49352 non-null int64\n",
      "terrace                   49352 non-null int64\n",
      "time                      49352 non-null int64\n",
      "tons                      49352 non-null int64\n",
      "train                     49352 non-null int64\n",
      "true                      49352 non-null int64\n",
      "tv                        49352 non-null int64\n",
      "unit                      49352 non-null int64\n",
      "utilities                 49352 non-null int64\n",
      "valet                     49352 non-null int64\n",
      "video                     49352 non-null int64\n",
      "view                      49352 non-null int64\n",
      "views                     49352 non-null int64\n",
      "virtual                   49352 non-null int64\n",
      "walk                      49352 non-null int64\n",
      "walls                     49352 non-null int64\n",
      "war                       49352 non-null int64\n",
      "washer                    49352 non-null int64\n",
      "water                     49352 non-null int64\n",
      "wheelchair                49352 non-null int64\n",
      "wifi                      49352 non-null int64\n",
      "windows                   49352 non-null int64\n",
      "work                      49352 non-null int64\n",
      "interest_level            49352 non-null int64\n",
      "dtypes: float64(9), int64(219)\n",
      "memory usage: 85.8 MB\n"
     ]
    }
   ],
   "source": [
    "train.info(verbose=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Target分布，分析是否均衡"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0,0.5,'Number of occurrences')"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAELCAYAAAARNxsIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAGx5JREFUeJzt3X2wHXWd5/H3h/AgPibIhcokYYKaGUUdo14hypSj6EDAHRMpUChHIkNtxAKF0rUEd5aMPMyM4ygrM8pupogkrkOIiBI1GDMMyII8JGB4CJHlioxcyZJgeNQVivDZP/p35eTm5N7OpU9OTu7nVdV1ur/96z7fwy340v379a9lm4iIiCbs0e0EIiJi95GiEhERjUlRiYiIxqSoREREY1JUIiKiMSkqERHRmBSViIhoTIpKREQ0JkUlIiIas2e3E9jZ9t9/f0+fPr3baURE9JTbbrvtEdt9o7Ubd0Vl+vTprFmzpttpRET0FEn/Uaddbn9FRERjOlZUJL1I0q2S7pC0TtLnS/xSSb+QtLYsM0tcki6SNCDpTklvaTnXPEn3lWVeS/ytku4qx1wkSZ36PRERMbpO3v56GjjC9lOS9gJukHR12fcZ21cMa380MKMshwEXA4dJ2g9YAPQDBm6TtNz2o6XNfOBmYAUwG7iaiIjoio5dqbjyVNncqywjzbM/B1hSjrsZmChpMnAUsMr25lJIVgGzy76X277J1fz9S4C5nfo9ERExuo72qUiaIGktsJGqMNxSdl1QbnFdKGmfEpsCPNhy+GCJjRQfbBNvl8d8SWskrdm0adML/l0REdFeR4uK7S22ZwJTgUMlvQE4G3gt8DZgP+CzpXm7/hCPId4uj4W2+2339/WNOiIuIiLGaKeM/rL9GHAdMNv2hnKL62ng68ChpdkgMK3lsKnAQ6PEp7aJR0REl3Ry9FefpIllfV/gvcDPSl8IZaTWXODucshy4KQyCmwW8LjtDcBK4EhJkyRNAo4EVpZ9T0qaVc51EnBVp35PRESMrpOjvyYDiyVNoCpey2x/X9K/S+qjun21Fji1tF8BHAMMAL8FTgawvVnSecDq0u5c25vL+seBS4F9qUZ9ZeRXREQXqRo4NX709/c7T9RH7NoO/6fDu53Cbu/GT9y4Q+0l3Wa7f7R2eaI+IiIak6ISERGNSVGJiIjGpKhERERjUlQiIqIxKSoREdGYFJWIiGhMikpERDQmRSUiIhqTohIREY1JUYmIiMakqERERGNSVCIiojEpKhER0ZgUlYiIaEyKSkRENCZFJSIiGpOiEhERjUlRiYiIxqSoREREY1JUIiKiMR0rKpJeJOlWSXdIWifp8yV+sKRbJN0n6XJJe5f4PmV7oOyf3nKus0v8XklHtcRnl9iApLM69VsiIqKeTl6pPA0cYftNwExgtqRZwBeAC23PAB4FTintTwEetf0a4MLSDkmHACcArwdmA1+TNEHSBOCrwNHAIcCJpW1ERHRJx4qKK0+Vzb3KYuAI4IoSXwzMLetzyjZl/3skqcSX2n7a9i+AAeDQsgzYvt/2M8DS0jYiIrqko30q5YpiLbARWAX8HHjM9rOlySAwpaxPAR4EKPsfB17ZGh92zPbiERHRJR0tKra32J4JTKW6snhdu2blU9vZt6PxbUiaL2mNpDWbNm0aPfGIiBiTnTL6y/ZjwHXALGCipD3LrqnAQ2V9EJgGUPa/AtjcGh92zPbi7b5/oe1+2/19fX1N/KSIiGijk6O/+iRNLOv7Au8F1gPXAseVZvOAq8r68rJN2f/vtl3iJ5TRYQcDM4BbgdXAjDKabG+qzvzlnfo9ERExuj1HbzJmk4HFZZTWHsAy29+XdA+wVNL5wE+BS0r7S4BvSBqgukI5AcD2OknLgHuAZ4HTbG8BkHQ6sBKYACyyva6DvyciIkbRsaJi+07gzW3i91P1rwyP/w44fjvnugC4oE18BbDiBScbERGNyBP1ERHRmBSViIhoTIpKREQ0JkUlIiIak6ISERGNSVGJiIjGpKhERERjUlQiIqIxKSoREdGYFJWIiGjMqEVF0ksk7VHW/0jS+yXt1fnUIiKi19S5UrkeeJGkKcA1wMnApZ1MKiIielOdoiLbvwWOBf7J9geo3gkfERGxlVpFRdLbgQ8DPyixTk6ZHxERPapOUTkTOBv4Tnm3yauoXrQVERGxlVGvOGz/GPixpJeU7fuBT3Y6sYiI6D11Rn+9vbytcX3ZfpOkr3U8s4iI6Dl1bn/9d+Ao4NcAtu8A3tnJpCIiojfVevjR9oPDQls6kEtERPS4OqO4HpT0DsCS9qbqT1nf2bQiIqIX1blSORU4DZgCDAIzy3ZERMRWRi0qth+x/WHbB9o+wPZf2v71aMdJmibpWknrJa2TdEaJ/42kX0laW5ZjWo45W9KApHslHdUSn11iA5LOaokfLOkWSfdJurxcSUVERJfUGf21WNLElu1JkhbVOPezwKdtvw6YBZwmaehJ/AttzyzLinLeQ4ATgNcDs4GvSZogaQLwVeBoqif5T2w5zxfKuWYAjwKn1MgrIiI6pM7trz+x/djQhu1HgTePdpDtDbZvL+tPUvXDTBnhkDnAUttP2/4FMAAcWpYB2/fbfgZYCsyRJOAI4Ipy/GJgbo3fExERHVKnqOwhadLQhqT92MFpWiRNpypEt5TQ6ZLulLSo5dxTgNZRZoMltr34K4HHbD87LB4REV1Sp6h8CfiJpPMknQf8BPiHul8g6aXAt4EzbT8BXAy8mqrDf0M5P4DaHO4xxNvlMF/SGklrNm3aVDf1iIjYQXU66pcAxwEPAxuBY21/o87Jy3tXvg180/aV5XwP295i+zngX6hub0F1pTGt5fCpwEMjxB8BJkrac1i83W9YaLvfdn9fX1+d1CMiYgzqvvnxZ8CVwFXAU5IOGu2A0udxCbDe9pdb4pNbmn0AuLusLwdOkLSPpIOBGcCtwGpgRhnptTdVZ/5y26aa2PK4cvy8kl9ERHTJqH0jkj4BLKC6UtlCddvJwJ+McujhwEeAuyStLbHPUY3emlnO8QDwMYAyA/Iy4B6qkWOn2d5ScjgdWAlMABbZXlfO91lgqaTzgZ9SFbGIiOiSOh3uZwB/XOfZlFa2b6B9v8eKEY65ALigTXxFu+PKjMmHDo9HRER31Ln99SDweKcTiYiI3lfnSuV+4DpJPwCeHgq29pNERERAvaLyy7LsXZaIiIi26rz58fMAkl5i+zedTykiInpV3vwYERGNyZsfIyKiMXnzY0RENCZvfoyIiMbkzY8REdGYEa9UyguyPmL7wzspn4iI6GEjXqmUubfm7KRcIiKix9XpU7lR0j8DlwO/f05l6K2OERERQ+oUlXeUz3NbYqZ6lW9ERMTvjdansgdwse1lOymfiIjoYaP1qTwHnL6TcomIiB5XZ0jxKkn/RdI0SfsNLR3PLCIiek6dPpW/Kp+tz6YYeFXz6URERC+rM0vxwTsjkYiI6H113lF/Uru47SXNpxMREb2szu2vt7Wsvwh4D3A7kKISERFbqXP76xOt25JeAXyjYxlFRETPqjX1/TC/BWaM1qiMFrtW0npJ6ySdUeL7SVol6b7yOanEJekiSQOS7pT0lpZzzSvt75M0ryX+Vkl3lWMukqQx/J6IiGhInTc/fk/S8rJ8H7gXuKrGuZ8FPm37dcAs4DRJhwBnAdfYngFcU7YBjqYqVjOA+cDF5fv3AxYAhwGHAguGClFpM7/luNk18oqIiA6p06fyjy3rzwL/YXtwtINsbwA2lPUnJa2nmj5/DvCu0mwxcB3w2RJfYtvAzZImSppc2q6yvRlA0ipgtqTrgJfbvqnElwBzgatr/KaIiOiAOkXll8AG278DkLSvpOm2H6j7JZKmA28GbgEOLAUH2xskHVCaTQFa3zA5WGIjxQfbxCMiokvq9Kl8C3iuZXtLidUi6aXAt4EzbT8xUtM2MY8h3i6H+ZLWSFqzadOm0VKOiIgxqlNU9rT9zNBGWd+7zskl7UVVUL5p+8oSfrjc1qJ8bizxQWBay+FTgYdGiU9tE9+G7YW2+2339/X11Uk9IiLGoE5R2STp/UMbkuYAj4x2UBmJdQmw3vaXW3YtB4ZGcM3j+U7/5cBJZRTYLODxcptsJXCkpEmlg/5IYGXZ96SkWeW7TqLeAIKIiOiQOn0qpwLfLC/qguoKoe1T9sMcDnwEuEvS2hL7HPD3wDJJp1D11xxf9q0AjgEGqIYtnwxge7Ok84DVpd25Q532wMeBS4F9qTro00kfEdFFdR5+/Dkwq/SNyPaTdU5s+wba93tA9VT+8PZm60krW/ctAha1ia8B3lAnn4iI6Lw6z6n8raSJtp8qQ4MnSTp/ZyQXERG9pU6fytG2HxvasP0o1W2qiIiIrdQpKhMk7TO0IWlfYJ8R2kdExDhVp6P+fwHXSPo61XMgf0X1JHxERMRW6nTU/4OkO4H3ltB5tld2Nq2IiOhFda5UAH4K7EV1pfLTzqUTERG9rM7orw8CtwLHAR8EbpF0XKcTi4iI3lPnSuW/Am+zvRFAUh/wb8AVnUwsIiJ6T53RX3sMFZTi1zWPi4iIcabOlcoPJa0ELivbH6KaUiUiImIrdUZ/fUbSscCfUk27stD2dzqeWURE9Jxao7/KtPVXjtowIiLGtfSNREREY1JUIiKiMdstKpKuKZ9f2HnpRERELxupT2WypD8D3i9pKcPejWL79o5mFhERPWekonIOcBbVu9+/PGyfgSM6lVRERPSm7RYV21cAV0j6b7bP24k5RUREj6rznMp5kt4PvLOErrP9/c6mFRERvajOhJJ/B5wB3FOWM0osIiJiK3UefnwfMNP2cwCSFlNNf392JxOLiIjeU/c5lYkt66/oRCIREdH76hSVvwN+KunScpVyG/C3ox0kaZGkjZLubon9jaRfSVpblmNa9p0taUDSvZKOaonPLrEBSWe1xA+WdIuk+yRdLmnvuj86IiI6Y9SiYvsyYBbV3F9XAm+3vbTGuS8FZreJX2h7ZllWAEg6BDgBeH055muSJkiaAHwVOBo4BDixtAX4QjnXDOBR4JQaOUVERAfVuv1le4Pt5bavsv1/ax5zPbC5Zh5zgKW2n7b9C2AAOLQsA7bvt/0MsBSYI0lUz8kMvShsMTC35ndFRESHdGPur9Ml3Vluj00qsSnAgy1tBktse/FXAo/ZfnZYvC1J8yWtkbRm06ZNTf2OiIgYZmcXlYuBVwMzgQ3Al0pcbdp6DPG2bC+03W+7v6+vb8cyjoiI2kYsKpL2aO1of6FsP2x7Sxme/C9Ut7egutKY1tJ0KvDQCPFHgImS9hwWj4iILhqxqJT/+N8h6aAmvkzS5JbNDwBDBWs5cIKkfSQdDMwAbgVWAzPKSK+9qTrzl9s2cC1wXDl+HnBVEzlGRMTY1Xn4cTKwTtKtwG+GgrbfP9JBki4D3gXsL2kQWAC8S9JMqltVDwAfK+daJ2kZ1RP7zwKn2d5SznM6sBKYACyyva58xWeBpZLOp3oY85I6PzgiIjqnTlH5/FhObPvENuHt/off9gXABW3iK4AVbeL38/zts4iI2AXUmVDyx5L+EJhh+98kvZjqqiEiImIrdSaU/M9Uz4P8zxKaAny3k0lFRERvqjOk+DTgcOAJANv3AQd0MqmIiOhNdYrK0+VpdgDKMN7tPhMSERHjV52i8mNJnwP2lfTnwLeA73U2rYiI6EV1ispZwCbgLqohwCuAv+5kUhER0ZvqjP56rkx5fwvVba97y8OHERERWxm1qEh6H/A/gJ9Tzbl1sKSP2b6608lFRERvqfPw45eAd9seAJD0auAHQIpKRERspU6fysahglLcD2zsUD4REdHDtnulIunYsrpO0gpgGVWfyvFUEz1GRERsZaTbX3/Rsv4w8GdlfRMwadvmEREx3m23qNg+eWcmEhERva/O6K+DgU8A01vbjzb1fUREjD91Rn99l2rK+u8Bz3U2nYiI6GV1isrvbF/U8UwiIqLn1SkqX5G0APgR8PRQ0PbtHcsqIiJ6Up2i8kbgI8ARPH/7y2U7Ypf0y3Pf2O0UxoWDzrmr2ynELqZOUfkA8KrW6e8jIiLaqfNE/R3AxE4nEhERva/OlcqBwM8krWbrPpUMKY6IiK3UKSoLxnJiSYuA/0Q1d9gbSmw/4HKqZ14eAD5o+1FJAr4CHAP8Fvjo0EAASfN4/v0t59teXOJvBS4F9qV6x8sZmZI/IqK7Rr39ZfvH7ZYa574UmD0sdhZwje0ZwDVlG+BoYEZZ5gMXw++L0ALgMOBQYIGkoSliLi5th44b/l0REbGTjVpUJD0p6Ymy/E7SFklPjHac7euBzcPCc4DFZX0xMLclvsSVm4GJkiYDRwGrbG+2/SiwCphd9r3c9k3l6mRJy7kiIqJL6rz58WWt25LmUl01jMWBtjeU826QdECJTwEebGk3WGIjxQfbxNuSNJ/qqoaDDjpojKlHRMRo6oz+2ort79L8Mypq91VjiLdle6Htftv9fX19Y0wxIiJGU2dCyWNbNvcA+hnhP+CjeFjS5HKVMpnnX/Y1CExraTcVeKjE3zUsfl2JT23TPiIiuqjOlcpftCxHAU9S9YGMxXJgXlmfB1zVEj9JlVnA4+U22UrgSEmTSgf9kcDKsu9JSbPKyLGTWs4VERFdUqdPZUzvVZF0GdVVxv6SBqlGcf09sEzSKcAvqd4iCdWQ4GOAAaohxSeX794s6Tyef9PkubaHOv8/zvNDiq8uS0REdNFIrxM+Z4TjbPu8kU5s+8Tt7HpPu5MBp23nPIuARW3ia4A3jJRDRETsXCNdqfymTewlwCnAK4ERi0pERIw/I71O+EtD65JeBpxBdVtqKfCl7R0XERHj14h9KuWJ9k8BH6Z6WPEt5SHEiIiIbYzUp/JF4FhgIfBG20/ttKwiIqInjTSk+NPAH1BN5vhQy1QtT9aZpiUiIsafkfpUdvhp+4iIGN9SOCIiojEpKhER0ZgUlYiIaEyKSkRENCZFJSIiGpOiEhERjUlRiYiIxqSoREREY1JUIiKiMSkqERHRmBSViIhoTIpKREQ0JkUlIiIak6ISERGNSVGJiIjGdKWoSHpA0l2S1kpaU2L7SVol6b7yOanEJekiSQOS7pT0lpbzzCvt75M0rxu/JSIintfNK5V3255pu79snwVcY3sGcE3ZBjgamFGW+cDFUBUhYAFwGHAosGCoEEVERHfsSre/5gCLy/piYG5LfIkrNwMTJU0GjgJW2d5s+1FgFTB7ZycdERHP61ZRMfAjSbdJml9iB9reAFA+DyjxKcCDLccOltj24hER0SXbfUd9hx1u+yFJBwCrJP1shLZqE/MI8W1PUBWu+QAHHXTQjuYaERE1deVKxfZD5XMj8B2qPpGHy20tyufG0nwQmNZy+FTgoRHi7b5voe1+2/19fX1N/pSIiGix04uKpJdIetnQOnAkcDewHBgawTUPuKqsLwdOKqPAZgGPl9tjK4EjJU0qHfRHllhERHRJN25/HQh8R9LQ9/+r7R9KWg0sk3QK8Evg+NJ+BXAMMAD8FjgZwPZmSecBq0u7c21v3nk/IyIihtvpRcX2/cCb2sR/DbynTdzAads51yJgUdM5RkTE2OxKQ4ojIqLHpahERERjujWkuCe89TNLup3Cbu+2L57U7RQiokG5UomIiMakqERERGNSVCIiojEpKhER0ZgUlYiIaEyKSkRENCZFJSIiGpOiEhERjUlRiYiIxqSoREREY1JUIiKiMSkqERHRmBSViIhoTIpKREQ0JkUlIiIak6ISERGNSVGJiIjGpKhERERjer6oSJot6V5JA5LO6nY+ERHjWU8XFUkTgK8CRwOHACdKOqS7WUVEjF89XVSAQ4EB2/fbfgZYCszpck4REeNWrxeVKcCDLduDJRYREV2wZ7cTeIHUJuZtGknzgfll8ylJ93Y0q+7aH3ik20nUpX+c1+0UdiU99bcDYEG7fwXHrZ76++mTO/y3+8M6jXq9qAwC01q2pwIPDW9keyGwcGcl1U2S1tju73YesePyt+tt+ftVev3212pghqSDJe0NnAAs73JOERHjVk9fqdh+VtLpwEpgArDI9roupxURMW71dFEBsL0CWNHtPHYh4+I2324qf7velr8fIHubfu2IiIgx6fU+lYiI2IWkqOwmMl1N75K0SNJGSXd3O5fYMZKmSbpW0npJ6ySd0e2cui23v3YDZbqa/wP8OdUw69XAibbv6WpiUYukdwJPAUtsv6Hb+UR9kiYDk23fLullwG3A3PH8716uVHYPma6mh9m+Htjc7Txix9neYPv2sv4ksJ5xPqtHisruIdPVRHSZpOnAm4FbuptJd6Wo7B5qTVcTEZ0h6aXAt4EzbT/R7Xy6KUVl91BrupqIaJ6kvagKyjdtX9ntfLotRWX3kOlqIrpAkoBLgPW2v9ztfHYFKSq7AdvPAkPT1awHlmW6mt4h6TLgJuCPJQ1KOqXbOUVthwMfAY6QtLYsx3Q7qW7KkOKIiGhMrlQiIqIxKSoREdGYFJWIiGhMikpERDQmRSUiIhqTohIREY1JUYkAJP2kRpszJb24w3nMHO05B0kflfTPDX9v4+eM8SlFJQKw/Y4azc4EdqiolNcS7IiZwLh+eC56W4pKBCDpqfL5LknXSbpC0s8kfVOVTwJ/AFwr6drS9khJN0m6XdK3yqSCSHpA0jmSbgCOl/RqST+UdJuk/y3ptaXd8ZLulnSHpOvLFDvnAh8qT2Z/qEbefZK+LWl1WQ6XtEfJYWJLuwFJB7Zr3/g/zBjX9ux2AhG7oDcDr6ealPNG4HDbF0n6FPBu249I2h/4a+C9tn8j6bPAp6iKAsDvbP8pgKRrgFNt3yfpMOBrwBHAOcBRtn8laaLtZySdA/TbPr1mrl8BLrR9g6SDgJW2XyfpKuADwNfLdz5g+2FJ/zq8PfC6F/jPK+L3UlQitnWr7UEASWuB6cANw9rMAg4BbqzmFGRvqvm7hlxejn8p8A7gW6UdwD7l80bgUknLgLHObvte4JCWc7+8vIHwcqqi9XWqCUYvH6V9RCNSVCK29XTL+hba/3siYJXtE7dzjt+Uzz2Ax2zPHN7A9qnlKuJ9wFpJ27SpYQ/g7bb/31bJSTcBr5HUB8wFzh+l/Ri+OmJb6VOJqO9JYOj/6m8GDpf0GgBJL5b0R8MPKC9s+oWk40s7SXpTWX+17VtsnwM8QvVOnNbvqONHVDNUU845s3yvge8AX6aalv3XI7WPaEqKSkR9C4GrJV1rexPwUeAySXdSFZnXbue4DwOnSLoDWAfMKfEvSrpL0t3A9cAdwLVUt6dqddQDnwT6Jd0p6R7g1JZ9lwN/yfO3vkZrH/GCZer7iIhoTK5UIiKiMemoj9hFSToZOGNY+Ebbp3Ujn4g6cvsrIiIak9tfERHRmBSViIhoTIpKREQ0JkUlIiIak6ISERGN+f+UQGoL+CmGNwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.countplot(train.interest_level)\n",
    "pyplot.xlabel('interest_level')\n",
    "pyplot.ylabel('Number of occurrences')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 每个样本分布不是很均衡"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = train['interest_level']\n",
    "\n",
    "remove_train = train.drop([\"interest_level\"], axis=1)\n",
    "X_train = np.array(remove_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 初步确定弱学习器数目"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#直接调用xgboost内嵌的交叉验证（cv），可对连续的n_estimators参数进行快速交叉验证\n",
    "#而GridSearchCV只能对有限个参数进行交叉验证\n",
    "def modelfit(alg, X_train, y_train, cv_folds=None, early_stopping_rounds=10):\n",
    "    xgb_param = alg.get_xgb_params()\n",
    "    xgb_param['num_class'] = 9\n",
    "    \n",
    "    #直接调用xgboost，而非sklarn的wrapper类\n",
    "    xgtrain = xgb.DMatrix(X_train, label = y_train)\n",
    "        \n",
    "    cvresult = xgb.cv(xgb_param, xgtrain, num_boost_round=alg.get_params()['n_estimators'], folds =cv_folds,\n",
    "             metrics='mlogloss', early_stopping_rounds=early_stopping_rounds)\n",
    "  \n",
    "    cvresult.to_csv('1_nestimators.csv', index_label = 'n_estimators')\n",
    "    \n",
    "    #最佳参数n_estimators\n",
    "    n_estimators = cvresult.shape[0]\n",
    "    \n",
    "    # 采用交叉验证得到的最佳参数n_estimators，训练模型\n",
    "    alg.set_params(n_estimators = n_estimators)\n",
    "    alg.fit(X_train, y_train, eval_metric='mlogloss')\n",
    "        \n",
    "    #Predict training set:\n",
    "    #train_predprob = alg.predict_proba(X_train)\n",
    "    #logloss = log_loss(y_train, train_predprob)\n",
    "\n",
    "   #Print model report:\n",
    "   # print (\"logloss of train :\" )\n",
    "   # print logloss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#params = {\"objective\": \"multi:softprob\", \"eval_metric\":\"mlogloss\", \"num_class\": 9}\n",
    "xgb1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=1000,  #数值大没关系，cv会自动返回合适的n_estimators\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel=0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "modelfit(xgb1, X_train, y_train, cv_folds = kfold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:1: FutureWarning: from_csv is deprecated. Please use read_csv(...) instead. Note that some of the default arguments are different, so please refer to the documentation for from_csv when changing your function calls\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEXCAYAAABCjVgAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XmcHHWd//HXp7vnvnLMJOQ+IFxBCBgJKEq8gV0BWUSjICCK7oKre6m4/lYXV9djV9FVQdQQD4RlxQNZVBCRcEPCHbJAyDnkmhyTzGTu7s/vj6qe6Uy6ZybJdNfM9Pv5eNSju46u+nRNT7+7rm+ZuyMiIgIQi7oAEREZORQKIiLSS6EgIiK9FAoiItJLoSAiIr0UCiIi0kuhIJLBzD5rZj+Mug6RqCgURhkzqzaz9Wb2/oxhNWa20cwuzBi20MzuNLPdZtZsZi+Y2ZfMbHw4/jIzS5pZa9itNbO/znPti82sMZ/LOBjZ6nH3L7v7h/O0vPVm9rZ8zDsfCvX3Gm3rZaxTKIwy7t4KXAl8y8wawsFfA1a4+y8AzOz1wJ+Bh4Bj3X0ccBbQA5yUMbtH3L3a3auBC4GvmdnJhXkncjDMLBF1DVIk3F3dKOyAZcAtwGJgJzAlY9yDwH8N8vrLgAf7DXsceH9G/7nAKqCZIGSOyxh3XDisOZzm3Ixx5wAvAC3Aq8A/AlVAO5ACWsNuao739V3gf8PXPwYcOYT1cSxwD7ALeBG46FDqAb4A/Cx83WzAgcuBTcBu4GPA64Bnw/f+nYzlHAn8Kfx77ABuBsaF434aLqs9XNanhrCO1wOfDpfVCSTC/lfD9/Ii8NYs6+I0YCsQzxj2buDZ8PmpwApgL7AN+EaOdboYaMwxrg74CdAEbAA+B8TCcXHgP8N1sA64OlyPiRzzWg+8Lce4jwBrwr/rHenPDGDAN4HtwJ5wHZ2Q6+8d9f/raOoiL0DdIf7hYDywJfzHuzxjeBWQBBYP8vrLyAiF8IuuGTg67D8a2Ae8HSgBPhX+c5aG/WuAz4b9bwn/AY8JX7sFeGNGnaeEz3N+yWTUsSz8Ajg1/BK8Gbh1kNdUEXxpXx6+5pRwvcw/2HrIHgo3AOXAO4AO4NfAJGBa+KV0Zjj9UeH6KgMagOXAdRnz3u/Lb6B1nDH908AMoAI4JnyfUzPqyxqYwCvA2zP6/wf4TPj8EeCS8Hk1cFqOeeT8exEEwm+AmrCOl4ArwnEfI/hSnh6u7z9yCKEQfq52hH/PMuC/gOXhuHcCK4FxBAFxHOEPo1x/b3VD67T7aJRy990EvzArgV9mjBpPsFtwa3qAmX0tPK6wz8w+lzHtaeHwVoKthJ8CL4fj3gv8r7vf4+7dwH8QfDG9nuCXaDXwFXfvcvc/AXcCS8LXdgPHm1mtu+929ycP8u390t0fd/ceglBYMMj0fwmsd/eb3L0nXN7tBLvEhqOeL7p7h7vfTfAlfou7b3f3V4EHgJMB3H1NuL463b0J+AZw5gDzHWgdp33b3Te5eztB2JeF76XE3de7+ys55n0L4d/DzGoIfj3fkrE+jjKzendvdfdHD2ZlmFk8rP0ad29x9/UEWwaXhJNcBHzL3RvDz+lXDmb+GT4ALHX3J929E7gGON3MZofvoYZgC9HcfbW7b8l4f4fz9y5qCoVRyswuJviF9kfgqxmjdhPsppiSHuDun/LguMKvCH5Jpz3q7uM8OKZwBDAf+HI4birBboH0PFIEv1KnheM2hcPSNoTjAP6K4Etog5ndb2anH+Tb25rxvI0ggAYyC1gUBlyzmTUTfKEcMUz1bMt43p6lvxrAzCaZ2a1m9qqZ7QV+BtQPMN+B1nHapozxa4BPEmzNbA+XNTXHvH8OXGBmZcAFwJPunl7WFQRbKf9nZk+Y2V8OUGM29QRbiBsyhmX+/adm1t3v+cHov35aCXbNTQt/iHyHYFfjNjO70cxqw0kP9+9d1BQKo5CZTSLYn/oR4KPARWb2JgB330ewH/6Cg5mnu28j+HX9rnDQZoIv2/QyjWA3xqvhuBlmlvn5mRmOw92fcPfzCHax/Bq4Lb2Yg6npIGwC7g8DLt1Vu/tfF7iefw/neaK71wIXE+zaSOu/vIHWcdbXuPvP3f2M8HXO/j8IMqd7geAL9Wzg/QQhkR73srsvIVgfXwV+YWZVQ3+b7CD4NT4rY1jv359g9830jHEzDmLemfqvnypgIn2fs2+7+2sJfswcDfxTODzX31uGQKEwOn0H+LW73xduMn8K+EH4q5Cw/0Nm9pkwQDCz6cCcXDM0s4kEByNXhYNuA/7CzN5qZiXAPxAc7HyYIHT2AZ8ysxIzW0wQJreaWamZfcDM6sJdInsJdntA8At7opnVDdN6SLsTONrMLgnrKTGz15nZcQWup4bgIHKzmU0j/JLKsA2Ym9E/0Do+gJkdY2ZvCf/OHQRbKcls04Z+Dvwt8CaCYwrp+VxsZg3hlklzODjnfMysPLMj2BK9DfhSeDr0LODvCbaM0u/rE2Y2zczGERwcH0xJv+UkwvovN7MF4Xv+MvCYu68P/76LwvW2L1wfyUH+3jIUUR/UUHdwHXA+wS+ocf2G3wt8KaN/EXAXwT99M/A88CVgYjj+MoJ/lvSZN9sJ9jlPypjHuwkOGO4B7ic8cBuOmx8O2xNO8+5weCnwe4LdWHuBJ4AzMl63lGAXQDO5zz76t4z+xQxycDqc7hiCM5aawvn/ieBYxEHVQ/YDzYmM6RvJOIhP8EX4uYx1sjJcn08TfMk3Zkx7HrAxXNY/DmEdr2f/A9MnEhz7aSE4GH9ntnWYMf1Mgi/w/+03/Gfh37uV4EfA+Tlevzh8//27owiOXf0sXN+bgH+h7+yjBMGW7E6Cs4/+jmDLwnIsZ32WZfxbOO5jBAfN0+93ejj8rQRnHLXSd6ZX9WB/b3WDdxauYBGRvDCzs4Eb3H3WoBNL5LT7SESGlZlVmNk5ZpYId6N9nuAkBxkFtKUgo4KZvRH4XbZxHpw9JSOEmVUS7Ao7luC4x/8Cn3D3vZEWJkOiUBARkV7afSQiIr1GXSNb9fX1Pnv27KjLEBEZVVauXLnD3RsGmy5voWBmMwjaRzmC4LS4G939W/2mMeBbBFcftgGX+SCXpM+ePZsVK1bkp2gRkTHKzDYMPlV+txR6gH9w9yfDtldWmtk9HlxpmXY2MC/sFgHXh48iIhKBvB1TcPct6V/97t4CrGb/Nl0guJjnJx54FBhnZlMQEZFIFORAc9iq4ckEzSNkmsb+jWU1cmBwiIhIgeQ9FMysmqChtU9mOU/ZsrzkgHNkzexKM1thZiuampryUaaIiJDnUAgbq7oduNndf5llkkb2b0FxOkG7Pvtx9xvdfaG7L2xoGPTguYiIHKK8hUJ4ZtGPgNXu/o0ck90BfNACpwF7vO9GGSIiUmD5PPvoDQR3YnrOzJ4Oh32WoOVG3P0GglY8zyG4BWEbwe0URUQkInkLBXd/kOzHDDKnceCqfNWQaeu2LaxZ/TQLFp5BdXVNIRYpIjLqFE0zF89dfyln/Pl9NG18KepSRERGrKIJhSMWXwlA577mQaYUESleRRMKpVXjAIWCiMhAiiYUyquC2/B2t+2JuBIRkZGraEKhonYCAMl23edDRCSXogmFyprxAKTataUgIpJL8YRCVW3wpENbCiIiuRRNKMQSCfZ5OXS1RF2KiMiIVTShANBmlcS6WqMuQ0RkxCqqUGiPVZLoViiIiORSVKHQEauipEehICKSS1GFQmeiitLkvqjLEBEZsYoqFHoS1ZSnFAoiIrkUVSgkS6qpSLVFXYaIyIhVdKFQ5QoFEZFciioUvKyWamunp6cn6lJEREakogoFKw+uat7XoquaRUSyKapQiIeh0NayK+JKRERGpuIKhYogFNpbdU8FEZFs8hYKZrbUzLab2fM5xteZ2W/N7BkzW2Vml+erlrREZXBPhQ6FgohIVvncUlgGnDXA+KuAF9z9JGAx8J9mVprHeigL777WvU/NZ4uIZJO3UHD35cBAO+8dqDEzA6rDafN6WlB5dbCl0KV7KoiIZBXlMYXvAMcBm4HngE+4eyrbhGZ2pZmtMLMVTU1Nh7zA8urgRjtJ3ZJTRCSrKEPhncDTwFRgAfAdM6vNNqG73+juC919YUNDwyEvsLI2ffc1nZIqIpJNlKFwOfBLD6wB1gHH5nOBVdV1pNygQweaRUSyiTIUNgJvBTCzycAxwNp8LtDiCVqsCuvQ7iMRkWwS+Zqxmd1CcFZRvZk1Ap8HSgDc/Qbgi8AyM3sOMODT7r4jX/WktVo1iS5tKYiIZJO3UHD3JYOM3wy8I1/Lz6UtXktpl7YURESyKaormgE6ErWU9+hAs4hINkUXCt2ldVSlFAoiItkUXSj0lI2jJtUSdRkiIiNS0YWCl4+nhja6urqjLkVEZMQpulCIVY4nZs7e5p1RlyIiMuIUXSjEqyYAsG/P9ogrEREZeYouFEprJgLQ1pz3SyJEREadoguF8tp6ADr2aveRiEh/RRcKlXVBg3qdrQoFEZH+ii4UqscHoZDap/s0i4j0V3ShUDMu2H3kbbsjrkREZOQpulCweAktVGIdCgURkf6KLhQgaCk13qmWUkVE+ivKUGiL11LarfaPRET6K8pQaOqpwNp1oFlEpL+iDIWyuslMNDWKJyLSX1GGQk/FRMb7Htw96lJEREaUogwFqhqotnb2tmhrQUQkU95CwcyWmtl2M3t+gGkWm9nTZrbKzO7PVy39xWuCC9iad2wp1CJFREaFfG4pLAPOyjXSzMYB3wPOdff5wHvyWMt+SmsnA9C6S6EgIpIpb6Hg7suBgU7xeT/wS3ffGE5fsLasK8cfAUD77q2FWqSIyKgQ5TGFo4HxZvZnM1tpZh/MNaGZXWlmK8xsRVNT02EvuHbiFAA6dU8FEZH9RBkKCeC1wF8A7wT+n5kdnW1Cd7/R3Re6+8KGhobDXnBdw1QAUq0KBRGRTIkIl90I7HD3fcA+M1sOnAS8lO8Fl5TX0E4ptk832hERyRTllsJvgDeaWcLMKoFFwOqCLNmMPTaOkg6FgohIprxtKZjZLcBioN7MGoHPAyUA7n6Du682s98DzwIp4IfunvP01eHWmhhHWZeauhARyZS3UHD3JUOY5uvA1/NVw0DaSyZQ1aFjCiIimYrzimagu3witSk1ny0ikqloQyFVWc9430NXdzLqUkRERoyiDYXHtscotSS7dx3+dQ8iImNF0YbCm04+AYDd2zdFXImIyMhRtKFQOXEaAK1NjRFXIiIychRtKIybPBOAjt0KBRGRtKIPhdQetZQqIpJWtKEQL6+hlUpirWopVUQkrWhDAWB3fAKl7duiLkNEZMQo6lBoLZ1EdZfaPxIRSSvqUOgsn0RdcmfUZYiIjBhFHQrJ6sk0+C46unqiLkVEZEQo6lCI1U6l1JI0bd8cdSkiIiNCUYdC2YTgArbmbRsjrkREZGQo6lCorp8BQNsOXcAmIgJFHgrjj9BVzSIimYo6FGrqZ5B0w5tfjboUEZERoahDgXgJu2ITKWlVKIiIQB5DwcyWmtl2Mxvwvstm9jozS5rZhfmqZSDNpZOp6lD7RyIikN8thWXAWQNNYGZx4KvAH/JYx4DaK6YysUdNXYiIQB5Dwd2XA7sGmezjwO3A9nzVMZhk7TQm+w5aO7qiKkFEZMSI7JiCmU0D3g3cMIRprzSzFWa2oqlpeG+fGR8/k1JLsvXVDcM6XxGR0SjKA83XAZ929+RgE7r7je6+0N0XNjQ0DGsRFQ2zAWjesnZY5ysiMholIlz2QuBWMwOoB84xsx53/3Uhixg/dS4AbU3rC7lYEZERadBQMLMjgUZ37zSzxcCJwE/cvflwFuzuczKWsQy4s9CBADBhShAKPbvU1IWIyFB2H90OJM3sKOBHwBzg54O9yMxuAR4BjjGzRjO7wsw+ZmYfO6yKh1msoo4Wqki06KpmEZGh7D5KuXuPmb0buM7d/8vMnhrsRe6+ZKhFuPtlQ502H3YlJlHRppZSRUSGsqXQbWZLgEuBO8NhJfkrqfBaK6cxoUuhICIylFC4HDgd+JK7rzOzOcDP8ltWYfXUzWaab2Nve2fUpYiIRGrQUHD3F9z9b939FjMbD9S4+1cKUFvBJCbOpdy62bxpfdSliIhEatBQMLM/m1mtmU0AngFuMrNv5L+0wqmaMg+A3Y0vRlyJiEi0hrL7qM7d9wIXADe5+2uBt+W3rMJqmHksAO3b1kRciYhItIYSCgkzmwJcRN+B5jGlatIceojhu9ZFXYqISKSGEgrXErRi+oq7P2Fmc4GX81tWgcVL2BGbRHmr2j8SkeI26HUK7v4/wP9k9K8F/iqfRUVhT8U0xrfpZjsiUtyGcqB5upn9KrxhzjYzu93MpheiuELqrJnNlNRWOroHbZ9PRGTMGsruo5uAO4CpwDTgt+GwMSW+ZwPjrZUNjWruQkSK11BCocHdb3L3nrBbBgxv+9UjQM2brgKgaf2Adw8VERnThhIKO8zsYjOLh93FwM58F1Zok+e+BoC2zasjrkREJDpDCYUPEZyOuhXYAlxI0PTFmFLWMJcuEthOXasgIsVrKM1cbHT3c929wd0nufv5BBeyjS2xONtKplPdojuwiUjxOtTbcf79sFYxQrRUz2FK1wZSKY+6FBGRSBxqKNiwVjFC+MR5TGc7r+44rJvKiYiMWocaCmPyp3T51ONJWIrN61+IuhQRkUjkDAUzazGzvVm6FoJrFsacSbNPAGDPhlURVyIiEo2coeDuNe5em6WrcfdBm8cws6XhVdBZT/w3sw+Y2bNh97CZnXQ4b2Q41EyfTwojtU1bCiJSnA5199FQLAPOGmD8OuBMdz8R+CJwYx5rGZrSSrYnplK9R/dVEJHilLdQcPflwK4Bxj/s7rvD3keBEdGeUnPN0UzrWkt3MhV1KSIiBZfPLYWDcQXwu6iLAPBJxzOLbazf0hR1KSIiBRd5KJjZmwlC4dMDTHOlma0wsxVNTfn9sq6ZdRIxcza//FRelyMiMhINpensbGchbQqb0557OAs3sxOBHwLnuXvO9pTc/UZ3X+juCxsa8tsW36SjXgtA26Zn87ocEZGRaNCziIBvAJuBnxNctPY+4AjgRWApsPhQFmxmM4FfApe4+0uHMo98KK2fS5uXsXOtthREpPgMJRTOcvdFGf03mtmj7n6tmX0214vM7BaCwKg3s0bg80AJgLvfAPwLMBH4npkB9Lj7wkN7G8MoFqOp8kiObV+PuxPWJiJSFIYSCikzuwj4Rdh/Yca4nFc2u/uSgWbq7h8GPjyE5RdcR/0JHLPxt2xpbmPq+KqoyxERKZihHGj+AHAJsD3sLgEuNrMK4Oo81haZ8lmvpcbaWfN/Oq4gIsVl0C0Fd18LvCvH6AeHt5yR4YhjToMHYe/aJ+D006MuR0SkYIZy9tH08Eyj7Wa2zcxuN7MRcaFZvpRNnU8XJcS2PhN1KSIiBTWU3Uc3AXcQNII3DfhtOGzsipewreIo6ltW4z4mG4QVEclqKKHQ4O43uXtP2C0D8nuxwAjQ0fAajvV1bNjRGnUpIiIFM5RQ2GFmF5tZPOwuBnJeaDZWVM9dRK218dKqlVGXIiJSMEMJhQ8BFwFbgS0Ep6Rens+iRoLJx78RgJY1j0RciYhI4QwaCu6+0d3PdfcGd5/k7ucDFxSgtkjF6ufRGquhcvuTUZciIlIwh9og3t8PaxUjUSzGjrrXMLfjBVo6uqOuRkSkIA41FIqi7QebsYh59iof/N49UZciIlIQhxoKRXGe5qT5byRmzpLOXww+sYjIGJDzimYzayH7l78BFXmraASpmHMaPcSJx+NRlyIiUhA5Q8HdawpZyIhUWsX26uOYvfdp9nX2UFU2lPYDRURGr8jvvDbSJWe+ntfYKzz5yuaoSxERyTuFwiAaXvM2Si1J47P3R12KiEjeKRQGUT739SSJYRseiLoUEZG8UygMpqyGppr5HL1vJU0tnVFXIyKSVwqFIYjNewsn2Ss8umpN1KWIiOSVQmEI6k86h7g5O569O+pSRETyKm+hYGZLwxvzPJ9jvJnZt81sjZk9a2an5KuWwxWbvpAWKqnadD/JVFFctyciRSqfWwrLgLMGGH82MC/srgSuz2MthyeeoHXqGzgj9gwr1++KuhoRkbzJWyi4+3JgoG/Q84CfeOBRYJyZTclXPYdr/MnvYqrt4tmVOgtJRMauKI8pTAM2ZfQ3hsMOYGZXmtkKM1vR1NRUkOL6Kz/uHFIYsZd+r1t0isiYFWUoZGtpNeu3rbvf6O4L3X1hQ0NEdwKtbmBV7BgWdj7KS9t0i04RGZuiDIVGYEZG/3RgRLclMecNF3JibB1/elw33hGRsSnKULgD+GB4FtJpwB533xJhPYOqPundAHQ/+ytSOgtJRMagvDX7aWa3AIuBejNrBD4PlAC4+w3AXcA5wBqgjdFw3+f6o9hdN5/Fu//Mig27OXXOhKgrEhEZVnkLBXdfMsh4B67K1/LzpWrh+zjx3v/HhTf/ll987tKoyxERGVa6ovkglZ70HlIYb+1ZTmdPMupyRESGlULhYNVOYc/k03hn6kHuW7096mpERIaVQuEQ1L5uCXNjW3ny0XujLkVEZFgpFA5BfP559FgJR2y8ky172qMuR0Rk2CgUDkXFOLrKJnBu7CFufujlqKsRERk2CoVDVPmeG6i3vex+/L9p79IBZxEZGxQKh2rum9kYm8FFqbu4feWmwacXERkFFAqHyowZdaWcFFvLow/8QVc4i8iYoFA4DPax5XQlanh7y6+5/+VoWm8VERlOCoXDUVZN/LWX8Bfxx7jlj4+pSW0RGfUUCocpvugjxEmyYMttPPLKzqjLERE5LAqFwzVhLqn5f8Wl8T/w6Z/ep60FERnVFArDIP7ma6iwbi5O/oq7ntsadTkiIodMoTAc6ufBiRdxaeIevn/Xw3R067oFERmdFArDJHbmpyizJOe33sbSh9ZFXY6IyCFRKAyXiUdiC5ZwScm9/OpPD7O9pSPqikREDppCYTgtvoZ4ooR/8mW885vLddBZREYdhcJwqptO7MxP8474ShZ0PM4dz2yOuiIRkYOS11Aws7PM7EUzW2Nmn8kyfqaZ3WdmT5nZs2Z2Tj7rKYjT/gavP4ZrS37MNbc9wfa92o0kIqNH3kLBzOLAd4GzgeOBJWZ2fL/JPgfc5u4nA+8DvpevegomUYqd83Vm2HauSvyGa375nHYjiciokc8thVOBNe6+1t27gFuB8/pN40Bt+LwOGBv7W+aeCSe+l4/Gfs2OFx/m+8vXRl2RiMiQ5DMUpgGZbUo3hsMyfQG42MwagbuAj2ebkZldaWYrzGxFU9MoaXju7K8Rr5vKD6q/z3d+/zQPqME8ERkF8hkKlmVY//0oS4Bl7j4dOAf4qZkdUJO73+juC919YUNDQx5KzYOKcdi7v09D92a+Xn0rH7/lKTbtaou6KhGRAeUzFBqBGRn90zlw99AVwG0A7v4IUA7U57Gmwpp9BnbGJzm7+27O7FrO2795P7v2dUVdlYhITvkMhSeAeWY2x8xKCQ4k39Fvmo3AWwHM7DiCUBhb+1kWfxZmns43y27kBH+FS5c+TktHd9RViYhklbdQcPce4GrgD8BqgrOMVpnZtWZ2bjjZPwAfMbNngFuAy3ysnaqTKIWLfkoMuLnk39i1ZT1X/HgFbV09UVcmInIAG23fwQsXLvQVK1ZEXcbB27YKfvQOmitnccbWv4OyWh769FuoqyyJujIRKQJmttLdFw42na5oLpTJ8+HCpYzb+yLLp11PqrOV079yry5uE5ERRaFQSEe/Ey74ARN2ruThmmvw7nbe+LX7WL1lb9SViYgACoXCO+ECOP8GxnVv4/Hqf2JqRQ8XfO9h7nx2bFy3JyKjm0IhCguWwPnXU9Ozi7snfJ3Tjkhx9c+f4trfvqAb9IhIpBQKUVnwfnjfzynZ+RJLe/6Zvz8Zlj60jpOvvYcXt7ZEXZ2IFCmFQpSOOQs++Buscw9/+8pHueNtzXQnU5x13XKu++NLdPZoq0FECkuhELWZi+DK+2HikZz44N/w7Bsf412vaeC6P77MSf96t9pMEpGCUiiMBONmwIf+ACdfTOVj1/Httmv47/dMwh0u+dHjXLr0cf5vq85QEpH8UyiMFCXlcN534cKbYOfLLPr9eTx/zjr++ayjeGrjbs667gFO+eI9PP/qnqgrFZExTKEw0pxwAfz1IzBzESV3X8NHVl/BQx+o4mNnHsmetm7+8r8e5OIfPsYDLzfp5j0iMuzUzMVI5Q6r74DbPwzJLljwAVrOuIafPd/F0ofW0dTSSXlJjE+fdSwXnDKdugo1lyEiuQ21mQuFwkjX2QoP/Ac8eB2YwRs+Qeeij/ObF9v419+uYl9nEjN4x/GTeddJU3nLsZOoLE1EXbWIjDAKhbFm11q478vw3C+grAYWfghO+2ue21PB7U82cvNjG+hOOjGDs0+YwluOncSZxzRQX10WdeUiMgIoFMaqrc8HWw4v/AZiCTjxvfD6j5OceDSPr9vFHc9s5hcrN9GdDP6uVWVxrnjDHE6dM5FTZo3TVoRIkVIojHW71sIj34UVS8FTMPN0OPkSmH8+qUQlL2zZy5/+bzvfX/4K+zqDi+ASMWP+tDoWzZnAwlnjOWFaHVPqyjHLdudUERlLFArFYt8OePpmePInsHMNWBxO/gDMvwBmvxHiCVo6unlyYzOPr9vJ4+t2sWL97t6bZSdixqK5E5g/tY75U2uZP7WWOfXVxGMKCpGxRKFQbNxh4yOw8sfw3G3B1kPlRDjuXDjuXTD7DEgExxc6upOs2ryHVZv3surVvdz57GbaupK9QWHAMUfUcNSkao6aVM28STXMqa9ixoQKasp1lpPIaKRQKGbd7fDyPbDqV/DCr4OAKKmCuWfCvLfD3DfD+NnB2UzplyRTrNneyqrNe3l5Wwtrtrfy8vZWNu5q22/WdRUlTB9fEXaVvY9T6sqZWF3KhKpSyhLxAr9hERnMiAgFMzsL+BYQB37o7l/JMs1FwBcAB55x9/cPNE+FwkHqaoP1D8DLdwdd88ZgeM1UmHV6cCxi1uuh4TiIHXgtY0d3kleaWlm3Yx+v7m4QCl+zAAAN5ElEQVSncXc7jbvbeHTtLjp7kqSyfHziMWPWhEomVpcysaqM+prgcUJVKTXlCWrKS6gNH2vKE9SWl1BdntAuK5E8ijwUzCwOvAS8HWgEngCWuPsLGdPMA24D3uLuu81skrtvH2i+CoXD4A47Xob1y2HDw0HXsiUYVz4uCIhpp8CUk2DKAqiZPMjsnF37uti0u53tezvY0drFztZOdu7rYkdrJztaO9nZ2sW6HfvoyZYe/cQMJteWZw2O3gCpKKGmLEF5SZzyklj4GDwvS4TDEsGwskSMmIJGBBh6KOTz/MRTgTXuvjYs6FbgPOCFjGk+AnzX3XcDDBYIcpjMoOHooHvdh4OQ2L0+OBax4SHY+Ci89Lu+6auPCAJi6oLg8YgToW56724nM2NidRkTh3AtRE8yRXN7Ny0dPbR09D3u7ehhb+/wjHGd3Ty+bhc9KSeZ8iGFSi7xmBEziJkxc0LlfoFSlohT1hsksd4wScSMRDxGIm7B81iMkngwLB6z4Hls/+lKYhnj4ulxfa8NxsV655eIW+/rFF4yUuQzFKYBmzL6G4FF/aY5GsDMHiLYxfQFd/99/xmZ2ZXAlQAzZ87MS7FFyQwmzAm6BeFeu469sPU52PJMX/fyH/peU1oD9fOg4RioPzp8PCY4RhHP/XFKxGPUV5cd1sV0Hd3J3uBo707S0Z2isydJZ3eKju4kHT3BsI7ujMdwfOd+44LnrZ09rNywm5Q7qVRwXCXqI2wGvTXEY4YByTAQE3HDMLpTqd7pyxIxZk2oIhYz4jGImwXP93sMAtGsLxxjRr9+g4xxwfT9+7O9PhwWO7jp09PCgfWQ7qdvWrNgXWTWlX14sBbDt9M73tLzoe999g7PmJ5+/WaZz4MJYtZvvuH09J9fenm9w/vm0Tt9vxrDsfvPL/0jDKgOd7fmUz5DIdtPn/7/cwlgHrAYmA48YGYnuHvzfi9yvxG4EYLdR8NfqvQqr4XZbwi6tK59sG1VEBA7XoKmF2Htn+GZW/qmsRjUToO6GUFT4ONmZjyfFWxhJA7v6ur0rqKGmvxfpe3et4XSk3J6kim6k8Gw7mSqd1jw6HSnUn3jkk5PKv0YDEtmTJcenvn6ZCpF0p1kClLhspMp733e9wiplIfTesa0HDBtUH8KTwbjUh68r5Tv379hZ1vvDZ0cKI0Hx5a6kilwKInHcII6gd5jP+mtt/RGTnpjLjPYZHhNqSvnkWvemtdl5DMUGoEZGf3Tgf53p28EHnX3bmCdmb1IEBJP5LEuOVilVTDj1KDL1LEnOEbR9CLsXgfNm2DPpuBYxbP/feB8qiZBzRFQMyXH4xFQ1QCx6M9eMgt3/URfyqi2fwg57gcGlGcMd4JHPAgZZ/9p+k/n6XFhfyoVvCZ9qNS9r9/pmzbzdQc8J5jYB3x9Xz3950vG9LlqIMu4vv6++aVnkp7m2Cm1+fpT9cpnKDwBzDOzOcCrwPuA/mcW/RpYAiwzs3qC3Ulr81iTDKfyOpi+MOj6S3bD3s1BSDRvCs562rMJWrcFB7c3PwX7shxCshhUTw4DYhJU1UPlBKisD667qAof08/Lavc7tVZGFjMjbhDPuuNARqK8hYK795jZ1cAfCI4XLHX3VWZ2LbDC3e8Ix73DzF4AksA/ufvOfNUkBRQvgfGzgi6XZDe0boeWrdC6NQiLln6P21ZB2w7o6cg+j1hJRkiEj+kAqRgXBFdZbbBbrPd5+DjAMRCRYqWL12Tkc4futqBJj7adQdf7PHzct3P//vbdg8+3pCoIiPLavrDYLzzSz/sPD/tLq7WVIqPGSDglVWR4mAXHNUqrBt7yyJTsgc69wXGPzr3BWVW9z/cE/Z17oaO573nbjqChwfR0ya5B6or1hUdZXUbA1PTVW1qd5Xl1v+HhsETp4a8rkcOkUJCxKZ4Ij0VMOPR5dHf0C5XmLKHS73nzJuhqCc7Y6toXbOEMVawkS5DkCpbM/spgq6e0Ekoqg+ElFX3P46XaopEhUyiI5FJSHnSDXNk9oFSyLyC69kFX6xCfZ/TvbTxwHgfD4mFQVPYFSEnFAGFSAYmK8P1XQqK83/D083B8SXkwPEszKTL6KBRE8ikWD49HDOOphKkU9LQHAdHZEmyNdLf3bZl0tUH3vn6P7RnP2/oCprUpGN7d3je9pwavIZt4WV9AlFRkCZiDCZscwZMeHldrvfmiUBAZbWKxvt1H1ZOGd97uwVlhPe1hkLQHZ351twW703qHd+w/TXd72B9O29Ox/+vbdmVMk/H6wY7b5FwHiYMMm/KBt3JKcoVSRXDRZRHtflMoiEgfs+CAd6I0OHCeb6nkAMGTK2yyBU/G9F2tsK8p+3wPiWUPi/5hkwh3NyYyupz9ZX2BU1JxYH+8LLLdcQoFEYlOLA5l1UGXb+4HbsEMupUzwNZPOmw69mQM64CezsPbCkqLl/UFSDpMXnsZnH7VsKyOXBQKIlIczPp+6RdCKgXJzjAwwqDo6cwIj4yutz9juu6M6dNdVUPey1YoiIjkQywGsQKG0DDROWQiItJLoSAiIr0UCiIi0kuhICIivRQKIiLSS6EgIiK9FAoiItJLoSAiIr1G3Z3XzKwJ2HCIL68HdgxjOaOV1kNA60HrIK0Y1sMsdx/0kuhRFwqHw8xWDOV2dGOd1kNA60HrIE3roY92H4mISC+FgoiI9Cq2ULgx6gJGCK2HgNaD1kGa1kOoqI4piIjIwIptS0FERAagUBARkV5FEwpmdpaZvWhma8zsM1HXU0hmtt7MnjOzp81sRThsgpndY2Yvh4/jo65zOJnZUjPbbmbPZwzL+p4t8O3ws/GsmZ0SXeXDK8d6+IKZvRp+Hp42s3Myxl0TrocXzeyd0VQ9vMxshpndZ2arzWyVmX0iHF50n4ehKIpQMLM48F3gbOB4YImZHR9tVQX3ZndfkHEu9meAe919HnBv2D+WLAPO6jcs13s+G5gXdlcC1xeoxkJYxoHrAeCb4edhgbvfBRD+T7wPmB++5nvh/85o1wP8g7sfB5wGXBW+12L8PAyqKEIBOBVY4+5r3b0LuBU4L+KaonYe8OPw+Y+B8yOsZdi5+3JgV7/Bud7zecBPPPAoMM7MphSm0vzKsR5yOQ+41d073X0dsIbgf2dUc/ct7v5k+LwFWA1Mowg/D0NRLKEwDdiU0d8YDisWDtxtZivN7Mpw2GR33wLBPw0wKbLqCifXey7Gz8fV4a6RpRm7Dsf8ejCz2cDJwGPo85BVsYSCZRlWTOfivsHdTyHYLL7KzN4UdUEjTLF9Pq4HjgQWAFuA/wyHj+n1YGbVwO3AJ91970CTZhk2ZtbDYIolFBqBGRn904HNEdVScO6+OXzcDvyKYJfAtvQmcfi4PboKCybXey6qz4e7b3P3pLungB/Qt4tozK4HMyshCISb3f2X4WB9HrIollB4AphnZnPMrJTgYNodEddUEGZWZWY16efAO4DnCd7/peFklwK/iabCgsr1nu8APhiedXIasCe9W2Es6rd//N0EnwcI1sP7zKzMzOYQHGh9vND1DTczM+BHwGp3/0bGKH0eskhEXUAhuHuPmV0N/AGIA0vdfVXEZRXKZOBXwf8FCeDn7v57M3sCuM3MrgA2Au+JsMZhZ2a3AIuBejNrBD4PfIXs7/ku4ByCA6ttwOUFLzhPcqyHxWa2gGCXyHrgowDuvsrMbgNeIDhj5yp3T0ZR9zB7A3AJ8JyZPR0O+yxF+HkYCjVzISIivYpl95GIiAyBQkFERHopFEREpJdCQUREeikURESkl0JBRER6KRREhsDMFvRrYvrc4WqC3cw+aWaVwzEvkcOl6xREhsDMLgMWuvvVeZj3+nDeOw7iNfExcmGZjDDaUpAxxcxmhzdT+UF4Q5W7zawix7RHmtnvw9ZjHzCzY8Ph7zGz583sGTNbHjaNci3w3vCmNO81s8vM7Dvh9MvM7PrwRi5rzezMsPXR1Wa2LGN515vZirCufw2H/S0wFbjPzO4Lhy2x4KZIz5vZVzNe32pm15rZY8DpZvYVM3shbO30P/KzRqXouLs6dWOmA2YTNNGwIOy/Dbg4x7T3AvPC54uAP4XPnwOmhc/HhY+XAd/JeG1vP8GNbG4laF3zPGAv8BqCH10rM2qZED7GgT8DJ4b964H68PlUgiYXGgiaJfkTcH44zoGL0vMCXqRva39c1Ote3djotKUgY9E6d0+3cbOSICj2Ezaj/Hrgf8L2cL4PpBuKewhYZmYfIfgCH4rfursTBMo2d3/Og1ZIV2Us/yIzexJ4iuDuZtnu/vc64M/u3uTuPcDNQLqp8yRBS58QBE8H8EMzu4CgjR6Rw1YUDeJJ0enMeJ4Esu0+igHN7r6g/wh3/5iZLQL+Ang6bDxuqMtM9Vt+CkiErY7+I/A6d98d7lYqzzKfbG35p3V4eBzBg0YeTwXeStDq79XAW4ZQp8iAtKUgRcmDm6ysM7P3QO/N2k8Knx/p7o+5+78AOwja1m8Bag5jkbXAPmCPmU0muOFRWua8HwPONLP68P7IS4D7+88s3NKp8+D+yp8kuGGOyGHTloIUsw8A15vZ54ASguMCzwBfN7N5BL/a7w2HbQQ+E+5q+veDXZC7P2NmTxHsTlpLsIsq7Ubgd2a2xd3fbGbXAPeFy7/L3bPd66IG+I2ZlYfT/d3B1iSSjU5JFRGRXtp9JCIivbT7SMY8M/suwd23Mn3L3W+Koh6RkUy7j0REpJd2H4mISC+FgoiI9FIoiIhIL4WCiIj0+v8YREWSqB0JSAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "cvresult = pd.DataFrame.from_csv('1_nestimators.csv')\n",
    "        \n",
    "# plot\n",
    "test_means = cvresult['test-mlogloss-mean']\n",
    "test_stds = cvresult['test-mlogloss-std'] \n",
    "        \n",
    "train_means = cvresult['train-mlogloss-mean']\n",
    "train_stds = cvresult['train-mlogloss-std'] \n",
    "\n",
    "x_axis = range(0, cvresult.shape[0])\n",
    "        \n",
    "pyplot.errorbar(x_axis, test_means, yerr=test_stds ,label='Test')\n",
    "pyplot.errorbar(x_axis, train_means, yerr=train_stds ,label='Train')\n",
    "pyplot.title(\"XGBoost n_estimators vs Log Loss\")\n",
    "pyplot.xlabel( 'n_estimators' )\n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig( 'n_estimators4_1.png' )\n",
    "\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 调整：max_depth & min_child_weight 参数\n",
    "(粗调，参数的步长为2；下一步是在粗调最佳参数周围，将步长降为1，进行精细调整)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': range(3, 10, 2), 'min_child_weight': range(1, 6, 2)}"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#max_depth 建议3-10， min_child_weight=1／sqrt(ratio_rare_event) =5.5\n",
    "max_depth = range(3,10,2)\n",
    "min_child_weight = range(1,6,2)\n",
    "param_test2_1 = dict(max_depth=max_depth, min_child_weight=min_child_weight)\n",
    "param_test2_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.60126, std: 0.00316, params: {'max_depth': 3, 'min_child_weight': 1},\n",
       "  mean: -0.60163, std: 0.00328, params: {'max_depth': 3, 'min_child_weight': 3},\n",
       "  mean: -0.60153, std: 0.00326, params: {'max_depth': 3, 'min_child_weight': 5},\n",
       "  mean: -0.58854, std: 0.00444, params: {'max_depth': 5, 'min_child_weight': 1},\n",
       "  mean: -0.58931, std: 0.00395, params: {'max_depth': 5, 'min_child_weight': 3},\n",
       "  mean: -0.58907, std: 0.00362, params: {'max_depth': 5, 'min_child_weight': 5},\n",
       "  mean: -0.59001, std: 0.00302, params: {'max_depth': 7, 'min_child_weight': 1},\n",
       "  mean: -0.58959, std: 0.00377, params: {'max_depth': 7, 'min_child_weight': 3},\n",
       "  mean: -0.58886, std: 0.00362, params: {'max_depth': 7, 'min_child_weight': 5},\n",
       "  mean: -0.60407, std: 0.00381, params: {'max_depth': 9, 'min_child_weight': 1},\n",
       "  mean: -0.60045, std: 0.00552, params: {'max_depth': 9, 'min_child_weight': 3},\n",
       "  mean: -0.59612, std: 0.00323, params: {'max_depth': 9, 'min_child_weight': 5}],\n",
       " {'max_depth': 5, 'min_child_weight': 1},\n",
       " -0.588538418194142)"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb2_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=232,  #第一轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch2_1 = GridSearchCV(xgb2_1, param_grid = param_test2_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch2_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch2_1.grid_scores_, gsearch2_1.best_params_,     gsearch2_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([129.74145999, 129.16750927, 126.57998729, 194.568153  ,\n",
       "        200.44058943, 190.28871274, 258.66064138, 265.13429971,\n",
       "        272.34465885, 352.48225694, 338.64077773, 296.51285853]),\n",
       " 'std_fit_time': array([ 3.71502345,  2.28371244,  6.0463592 ,  7.98814722, 13.22775018,\n",
       "         8.41172607,  5.16000861,  7.35666827,  5.41002751,  5.69011774,\n",
       "        10.04543979, 21.75070197]),\n",
       " 'mean_score_time': array([0.5687459 , 0.60097637, 0.51709633, 0.71268401, 0.76973925,\n",
       "        0.71828985, 1.11066599, 1.1020575 , 1.0403986 , 2.08740349,\n",
       "        1.35540094, 0.99525528]),\n",
       " 'std_score_time': array([0.07937912, 0.08449624, 0.08199942, 0.02812007, 0.11037668,\n",
       "        0.05105378, 0.04167458, 0.09882867, 0.03130694, 0.31159226,\n",
       "        0.17689195, 0.06802532]),\n",
       " 'param_max_depth': masked_array(data=[3, 3, 3, 5, 5, 5, 7, 7, 7, 9, 9, 9],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_min_child_weight': masked_array(data=[1, 3, 5, 1, 3, 5, 1, 3, 5, 1, 3, 5],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'max_depth': 3, 'min_child_weight': 1},\n",
       "  {'max_depth': 3, 'min_child_weight': 3},\n",
       "  {'max_depth': 3, 'min_child_weight': 5},\n",
       "  {'max_depth': 5, 'min_child_weight': 1},\n",
       "  {'max_depth': 5, 'min_child_weight': 3},\n",
       "  {'max_depth': 5, 'min_child_weight': 5},\n",
       "  {'max_depth': 7, 'min_child_weight': 1},\n",
       "  {'max_depth': 7, 'min_child_weight': 3},\n",
       "  {'max_depth': 7, 'min_child_weight': 5},\n",
       "  {'max_depth': 9, 'min_child_weight': 1},\n",
       "  {'max_depth': 9, 'min_child_weight': 3},\n",
       "  {'max_depth': 9, 'min_child_weight': 5}],\n",
       " 'split0_test_score': array([-0.59620456, -0.5964919 , -0.59647164, -0.58081778, -0.58244275,\n",
       "        -0.58290437, -0.58494029, -0.58367199, -0.58312698, -0.59842957,\n",
       "        -0.59339405, -0.59125862]),\n",
       " 'split1_test_score': array([-0.60043246, -0.5999296 , -0.60020021, -0.58661085, -0.58743063,\n",
       "        -0.58787314, -0.5887725 , -0.58871957, -0.5870892 , -0.60224844,\n",
       "        -0.59604512, -0.59537044]),\n",
       " 'split2_test_score': array([-0.60104356, -0.60183053, -0.60100556, -0.59030368, -0.59159449,\n",
       "        -0.58920255, -0.59110788, -0.59246147, -0.58953815, -0.6063131 ,\n",
       "        -0.60172562, -0.59676801]),\n",
       " 'split3_test_score': array([-0.60588541, -0.60607921, -0.60581736, -0.59171307, -0.59181353,\n",
       "        -0.59242864, -0.59126997, -0.58842838, -0.59058356, -0.60360949,\n",
       "        -0.6016831 , -0.59586793]),\n",
       " 'split4_test_score': array([-0.60274905, -0.60379552, -0.60417106, -0.59324813, -0.5932821 ,\n",
       "        -0.59291896, -0.59397059, -0.59465229, -0.59395711, -0.60974293,\n",
       "        -0.60941008, -0.60135645]),\n",
       " 'mean_test_score': array([-0.60126292, -0.60162522, -0.601533  , -0.58853842, -0.58931246,\n",
       "        -0.5890653 , -0.590012  , -0.58958643, -0.58885869, -0.60406836,\n",
       "        -0.60045105, -0.59612397]),\n",
       " 'std_test_score': array([0.00315921, 0.0032808 , 0.0032554 , 0.00444295, 0.00394855,\n",
       "        0.00362103, 0.00302359, 0.00376825, 0.00361692, 0.00380928,\n",
       "        0.00552458, 0.0032278 ]),\n",
       " 'rank_test_score': array([ 9, 11, 10,  1,  4,  3,  6,  5,  2, 12,  8,  7]),\n",
       " 'split0_train_score': array([-0.57869666, -0.57992735, -0.57995   , -0.51032136, -0.51745076,\n",
       "        -0.52056737, -0.40691814, -0.42773923, -0.44192299, -0.28076561,\n",
       "        -0.32600501, -0.35338763]),\n",
       " 'split1_train_score': array([-0.57771734, -0.57831986, -0.57897076, -0.50977527, -0.51565318,\n",
       "        -0.51785846, -0.40451978, -0.42650088, -0.43783581, -0.27753519,\n",
       "        -0.32470107, -0.35335643]),\n",
       " 'split2_train_score': array([-0.57782233, -0.57867199, -0.57821996, -0.50912895, -0.51451856,\n",
       "        -0.5178639 , -0.40307153, -0.42404854, -0.43695503, -0.27953085,\n",
       "        -0.32572616, -0.3518719 ]),\n",
       " 'split3_train_score': array([-0.57685825, -0.57814649, -0.57866503, -0.50983541, -0.51509483,\n",
       "        -0.51810529, -0.40277879, -0.42462354, -0.43729477, -0.27761508,\n",
       "        -0.32367993, -0.35283232]),\n",
       " 'split4_train_score': array([-0.57566933, -0.57697725, -0.57757356, -0.50736342, -0.51297575,\n",
       "        -0.51644029, -0.40387505, -0.42589183, -0.439729  , -0.28227734,\n",
       "        -0.3280338 , -0.35452916]),\n",
       " 'mean_train_score': array([-0.57735278, -0.57840859, -0.57867586, -0.50928488, -0.51513861,\n",
       "        -0.51816706, -0.40423266, -0.4257608 , -0.43874752, -0.27954481,\n",
       "        -0.32562919, -0.35319549]),\n",
       " 'std_train_score': array([0.00102352, 0.00094882, 0.00079164, 0.00103288, 0.00146117,\n",
       "        0.00133673, 0.0014754 , 0.00131941, 0.00185481, 0.00182865,\n",
       "        0.0014556 , 0.00086316])}"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch2_1.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.588538 using {'max_depth': 5, 'min_child_weight': 1}\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZoAAAELCAYAAADgPECFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xl8VPW5+PHPM0vWmQQIO4godUFBQKMgsgXFBSGUq3WrvWix1Gqv1/ZqwdrbXu+1Lbb9qV2sra1trW3VbkqgiGyJ4II0YESE1q2ICLIJSSb7zDy/P85JCGEm+2QSeN6v17zO9j3nPHMg55nvOd9zvqKqGGOMMYniSXYAxhhjjm+WaIwxxiSUJRpjjDEJZYnGGGNMQlmiMcYYk1CWaIwxxiSUJRpjjDEJZYnGGGNMQlmiMcYYk1C+ZO1YRPoAzwDDgR3ANap6KEa5CPCmO7lTVfPd+RcD38dJliHgJlV9V0SGAU8AvQAvsEhVlzcXS9++fXX48OGd8K2MMebEsWnTpgOq2q+lcpKsV9CIyPeAT1R1sYgsAnqr6sIY5UKqGogx/21gjqpuF5HbgAtU9SYReQx4XVUfFZGzgOWqOry5WHJzc7W4uLhTvpcxxpwoRGSTqua2VC6Zl87m4NQ8cIefbuP6CmS549nA7hbmG2OMSYKkXToDBqjqHgBV3SMi/eOUSxORYiAMLFbV59z5twDLRaQKKAMmuPP/B1gpIv8BZAKXxNqoiCwAFgAMGzasE76OMcaYWBJaoxGR1SKyNcZnThs2M8ytmt0APCwiI9z5XwFmqupQ4NfAg+7864HfuPNnAk+KyDHfU1UfU9VcVc3t16/FS4zGGGPaKaE1GlWNWZsAEJG9IjLIrc0MAvbF2cZud/i+iBQB40SkDBijqq+5xZ4BVrjj84HL3XVeFZE0oG+87RtjjEmsZN6jKQDmuePzgCVNC4hIbxFJdcf7AhcB24BDQLaInO4WnQFsd8d3Ahe764wE0oD9CfoOxhhjWpDMezSLgT+KyHyc5PAZABHJBW5V1VuAkcDPRSSKkxQXq+o2t9wXgL+4yw4Bn3e3+1/AL0TkKzgNA25S693NGGOSJmnNm7sTa95sjDFt19rmzcms0fR8n7wP766B9N6Qlg1pvZxhei9n3JeS7AiNMSbpLNF0xEebYfld8Zf70t2k4yahxuMNCSlGgkrLhtQgiHTddzHGmASxRNMRZ82B4ZOh+jBUHYbqUme8utSdPnz0srKPYN82qCqFmtLmty2e+EkoZvLqffQyr79rjoExxrTAEk1HeP0QHOB82ioagZqyOAmqNHbyKv3oyHSktvnt+zObr0E1tywl02pTxphOY4kmWTxe595Oeu+2r6sKdVWtT1BVh6F0F+zd6ozXlrcQm699CSq9N6Rmgdf+WxnT3dRF6wjVhiivLae8rpzy2nJCtSFOCp7EGX3OSOi+7YzQE4lASobzyRrU9vUjYac21TQpNVe7OvTBkfFouPntpwTbmKAajfvTrTZlTBOqSlW4ykkOdU6yKKsti5k4ymvLKas7sixUG6K8rpyqcFXMbd886mZLNCYBvD7I6ON82koV6ipbn6CqDsOhHbDHnVcban77Hn8rElScxhRp2U5N0ZhuJhKNNCSIhk+T5FA/XT+vrLbsqHUiGml2H36Pn2BK0Pn4gwRSAvTP6E9WShYBf4BgijOv8XQwJUj/jHivmew8lmhM24g493BSMiF7SNvXj9RBdePa1OHmL/9VHnSakddPt/DHRqqbcNIbJ6BezSeo+mX+tPYdE3NcU1VqIjUtJoeGJNE0odSWUxmubHE/mb4MAr4Mgv4Mgr50+nnTOTWzF4GsVLI8qQS8qQQ9KQTFT9DjIyh+AnjJEi8BPKRGo4iGnb+xSB1E66C2DqprIFIOkZ3O1YhIrbvcLXvWHBj32YQeQ0s0pmt5/ZCZ43zaStWpEcVt4Vd67LKD7x0Zr2vhj92b2rYa1FHN0bPAYx3Wdplo5MjJtPGJNcZ4NFJDqLacUENNIkR5XYhQXSVldRWUhysJRaopD1dRHqmmPFJDeaSGULSWcq2lPBqmjmiz4fgUAghBFYIKQYXh0SgBVYKRKMFohGAkTDAcJhAJkxWpIxCNEnQ/gajS6XVx8Tp/b94U576r1+9cMfD6jx6vrejsPR/DEo3pOUSc54tSg8BJbV8/XNtyE/TGyyr2wcF3jiyjubdoCKRltSJBxald+VLbeVDaQfXIr9lWnKgbxo/5NVzbaLk73dJ4tC7ONup/idceM14bqaOcMOUaJqQRyohSLhDyeChv9HGmhTKvh5B4KHeHIY+gLdz3S29y0u8djTLMTRhBFQJ43JqDl6B4CYrPrVn4CXhTSPekIA0ndb9zebrxuMddFnO8STKIN97iNholD4+/W/3wsURjThy+FAj0cz5tFY06rfVaqkE1Xrb/n0eWhatbiC09doJKzQKNxjhRxxpvLkk0mZdwAt4U1Oun0uuj3JdCuc9PucdLuc/nDD1OEij3eCjzQMgL5aKEUMqAcnyEVKhp4be+ByHgSSHoTXM+vjSG+jII+NLJ8mUS8GcSTAkQbLgvkUUgJUhWajaB1GwCqb3w+9ObnKh91iilE1miMaY1PPUP0GZDr3Z0lFdXHb/BxFH3q9xl5Xtg/3bnfpZ4Wr4E4ktzanrH/Lpt/MvaH3vd+hOrN+WY8To8hKijPBqmPFpHebSGULTOvcTkfsJV7qeCUF1lw6Wp+vsVUW162Snsfhyp3tSjbmIHU4IMSgkcNR3vRnYwJUiGLwOxpNCtWaIxpiv405xPex7ubadYTWJjt3o6RHl125rE1hPkqBZNwZQgAzMHcVqc5FA/HUwJNsxL8do7AY93lmiM6YbqWzpVh6sbkkSozmnymqwmsY2TQ/0n05+J59gObI05iiUaY1ohqlFqIjXUhGuojlRTHa52EkGkumFefWKoidRQE6mhKlx11DqNlzesF65uWNa4XE2kplVxZfozG07+WSlZ9Mvox6kppxLwN0oOTS5DNU4cqd4ubIRgTliWaEyPFNXoUSf1+hN203lNk0HDvCbL4pZz59VGW3i3XDNSvamkelNJ86aR6kslzZfmjHtTyU7LZoB3gLPcl9ZQrmHcl0bAH4hZqwj4A3jtAVXTAyQl0YhIH+AZYDiwA7hGVQ/FKBcB3nQnd6pqvjt/OvADIAXYBMxX1bA4dwR/CMwEKnF619yc2G9jwHnyuaWTddxf/Y1/ycf69R/jV39dB1pO1Z/wjzr5uyf3Xmm9jpqX6k09arwhSTSZV59M0n3pR81L8abYpSVzwktWjWYRsEZVF4vIInd6YYxyVao6tvEMEfEATwAXq+rbIvK/wDzgceAK4DT3Mx541B2ecMLRcMu/6hv/mm80rz2/+sMtvf8sDkGOOlE3/lWf6kult793w8k71vKjTu5xEkhDgvClkeJJsRZKxnSxZCWaOcA0d/wJoIjYiSaWHKBGVd92p1cB9+AkmjnAb9Xpn3qDiPQSkUGquqezAm+vumhdzF/uDb/WY5zIj/o138Zf/WHt2Ik/3q/6HH9Os8uPSgaNLhHFShB24jfmxJCsRDOg/uSvqntEJN5b3dJEpBin0f1iVX0OOAD4RSRXVYuBqznymPgQ4MNG6+9y5yUk0ZTsK+HJbU+26ld/S61/4vGIJ+Ylnvp5wYxgs8tjJYPGCaBhuTv0e/x24jfGdKqEJRoRWQ0MjLHo3jZsZpiq7haRU4G1IvKmqr4nItcBD4lIKrCSI09/xTpDxnxviIgsABYADBvWjgfwgFBdiLcPvX3k0o4vlezU7GOu29ef2Ju7RNT0BnD9Mp/HZyd+Y0yPlrBEo6qXxFsmInvrL2mJyCBgX5xt7HaH74tIETAOeE9VXwUmu9u6FDjdXWUXR78EayiwO862HwMeA8jNzW3uJVZxTRoyiUlzJ7VnVWOMOWEkqzlMAc4NfNzhkqYFRKS3W2NBRPoCFwHb3On+7jAV597Ozxpt99/FMQEo7Q73Z4wx5kSWrESzGJghIu8AM9xpRCRXRH7plhkJFIvIG0Ahzj2abe6yu0VkO7AFWKqqa935y4H3gXeBXwC3dcm3McYYE5c4DbRObLm5uVpcXJzsMIwxpkcRkU2qmttSOXuSzBhjTEJZojHGGJNQlmiMMcYklCUaY4wxCWWJxhhjTEJZojHGGJNQlmiMMcYklCUaY4wxCWWJxhhjTEJZojHGGJNQlmiMMcYklCUaY4wxCWWJxhhjTEJZojHGGJNQlmiMMcYklCUaY4wxCWWJxhhjTEIlJdGISB8RWSUi77jD3nHKRUSkxP0UNJo/XUQ2i8hWEXlCRHzu/M+KyBb384qIjOmq72SMMSa2ZNVoFgFrVPU0YI07HUuVqo51P/kAIuIBngCuU9VRwAfAPLf8v4CpqnoO8H/AY4n8EsYYY1qWrEQzBydZ4A4/3YZ1c4AaVX3bnV4FXAWgqq+o6iF3/gZgaCfEaowxpgOSlWgGqOoeAHfYP065NBEpFpENIlKfjA4AfhHJdaevBk6Kse584Pl4AYjIAnfbxfv372/ftzDGGNMiX6I2LCKrgYExFt3bhs0MU9XdInIqsFZE3lTV90TkOuAhEUkFVgLhJvvOw0k0k+JtWFUfw720lpubq22IyRhjTBskLNGo6iXxlonIXhEZpKp7RGQQsC/ONna7w/dFpAgYB7ynqq8Ck91tXQqc3mjb5wC/BK5Q1YOd9X2MMca0T7IunRVw5Ab+PGBJ0wIi0tutsSAifYGLgG3udH93mAosBH7mTg8D/gp8rtE9HGOMMUmUrESzGJghIu8AM9xpRCRXRH7plhkJFIvIG0AhsFhVt7nL7haR7cAWYKmqrnXnfxOnscBP3SbRxV30fYwxxsQhqnZ7Ijc3V4uLLScZY0xbiMgmVc1tqZy9GcAYY0xCWaIxxhiTUJZojDHGJJQlGmOMMQllicYYY0xCWaIxxhiTUJZojDHGJJQlGmOMMQllicYYY0xCWaIxxhiTUJZojDHGJJQlGmOMMQllicYYY0xCWaIxxhiTUJZojDHGJJQlGmOMMQnlS8ZORaQP8AwwHNgBXKOqh2KUiwBvupM7VTXfnT8d+AGQAmwC5qtquNF65wMbgGtV9c/tibGuro5du3ZRXV3dntVND5SWlsbQoUPx+/3JDsWY40pSEg2wCFijqotFZJE7vTBGuSpVHdt4hoh4gCeAi1X1bRH5X2Ae8Li73As8ALzQkQB37dpFMBhk+PDhiEhHNmV6AFXl4MGD7Nq1i1NOOSXZ4RhzXEnWpbM5OMkCd/jpNqybA9So6tvu9CrgqkbL/wP4C7CvIwFWV1eTk5NjSeYEISLk5ORYDdaYBEhWohmgqnsA3GH/OOXSRKRYRDaISH0yOgD4RaS+n+qrgZMARGQIMBf4WWcEaUnmxGL/3sYkRsIunYnIamBgjEX3tmEzw1R1t4icCqwVkTdV9T0RuQ54SERSgZVA/f2Zh4GFqhpp6aQhIguABQDDhg1rQ0jGGGPaImE1GlW9RFVHxfgsAfaKyCAAdxjzMpeq7naH7wNFwDh3+lVVnayqFwDrgHfcVXKBp0VkB05N56eNakJNt/2Yquaqam6/fv0662t3msOHD/PTn/60Xes+/PDDVFZWdnJEnWPatGkUFxe3a93nnnuObdu2tWlb1dXVXHDBBYwZM4azzz6bb33rW+3atzGm/ZJ16awA5wY+7nBJ0wIi0tutsSAifYGLgG3udH93mIrTiOBnAKp6iqoOV9XhwJ+B21T1ucR+lcQ4XhNNRzRNNK2RmprK2rVreeONNygpKWHFihVs2LAhQREaY2JJVquzxcAfRWQ+sBP4DIB73+VWVb0FGAn8XESiOAlxsarWn2XuFpFZ7vxHVXVtIoO9b+lbbNtd1qnbPGtwFt+afXbc5YsWLeK9995j7NixzJgxg/79+/PHP/6Rmpoa5s6dy3333UdFRQXXXHMNu3btIhKJ8N///d/s3buX3bt3k5eXR9++fSksLIy5/UAgwO23387q1avp3bs33/nOd/ja177Gzp07efjhh8nPz2fHjh187nOfo6KiAoCf/OQnTJw4kWeffZZHHnmEVatW8fHHHzN16lTWrVvHwIHHXimtqqri5ptvZtu2bYwcOZKqqqqGZStXruRb3/oWNTU1jBgxgl//+tcEAgGGDx/Otdde2xD7H/7wB/bt20dBQQEvvvgi999/P3/5y18A+NOf/sRtt93G4cOHefzxx5k8efJR+xcRAoEA4DRZr6urs3sxxnSxpNRoVPWgql6sqqe5w0/c+cVukkFVX1HV0ao6xh0+3mj9u1V1pKqeoaoPx9nHTe19hqY7WLx4MSNGjKCkpIQZM2bwzjvvsHHjRkpKSti0aRPr1q1jxYoVDB48mDfeeIOtW7dy+eWXc8cddzB48GAKCwvjJhmAiooKpk2bxqZNmwgGg3zjG99g1apVPPvss3zzm98EoH///qxatYrNmzfzzDPPcMcddwAwd+5cBg4cyCOPPMIXvvAF7rvvvphJBuDRRx8lIyODLVu2cO+997Jp0yYADhw4wP3338/q1avZvHkzubm5PPjggw3rZWVlsXHjRr785S9z5513MnHiRPLz8/n+979PSUkJI0aMACAcDrNx40Yefvhh7rvvPgB2797NzJkzG7YViUQYO3Ys/fv3Z8aMGYwfP74D/zLGmLZKVo2mR2mu5tEVVq5cycqVKxk3bhwAoVCId955h8mTJ3PXXXexcOFCZs2adcyv+eakpKRw+eWXAzB69GhSU1Px+/2MHj2aHTt2AE4N4Mtf/jIlJSV4vV7efvvthvV//OMfM2rUKCZMmMD1118fdz/r1q1rSFDnnHMO55xzDgAbNmxg27ZtXHTRRQDU1tZy4YUXNqxXv83rr7+er3zlK3G3/2//9m8AnHfeeQ1xDx48mOXLlzeU8Xq9lJSUcPjwYebOncvWrVsZNWpUq46TMabjLNH0AKrKPffcwxe/+MVjlm3atInly5dzzz33cOmllzbURlri9/sbLiF5PB5SU1MbxsNhpxHfQw89xIABA3jjjTeIRqOkpaU1rP/RRx/h8XjYu3cv0WgUjyd+5TjWpSpVZcaMGTz11FMtrtPcpa76uL1eb0Pc8fTq1Ytp06axYsUKSzTGdKE2XToTEY+IZCUqGHNEMBikvLwcgMsuu4xf/epXhEIhwDnJ79u3j927d5ORkcGNN97IXXfdxebNm49ZtyNKS0sZNGgQHo+HJ598kkgkAjiXq26++Wb+8Ic/MHLkyKMueTU1ZcoUfv/73wOwdetWtmzZAsCECRN4+eWXeffddwGorKw8qsb0zDPPNAzrazrt+V779+/n8OHDgHO/aPXq1Zx55plt2oYxpmNarNGIyB+AW4EIznvFskXkQVX9fqKDO5Hl5ORw0UUXMWrUKK644gpuuOGGhhNuIBDgd7/7He+++y533303Ho8Hv9/Po48+CsCCBQu44oorGDRoULP3aVpy2223cdVVV/GnP/2JvLw8MjMzAfjOd77D5MmTmTx5MmPHjuX888/nyiuvZOTIkcds40tf+hI333wz55xzDmPHjuWCCy4AoF+/fvzmN7/h+uuvp6amBoD777+f008/HYCamhrGjx9PNBptqPVcd911fOELX+BHP/oRf/5z/Ntvu3fv5pZbbmH58uXs2bOHefPmEYlEiEajXHPNNcyaNavdx8QY03aiqs0XEClR1bEi8lngPJzmxJtU9ZyuCLAr5ObmatPnMbZv3x7zxGkSb/jw4RQXF9O3b98u37f9uxvTeiKySVVzWyrXmktnfhHx47yPbImq1gHNZydjjDHG1ZrGAD/HeZX/G8A6ETkZ6NyHSkzCjB8/vuHSVL0nn3yS0aNHd+p+XnjhBRYuPPoF3KeccgrPPvtsm7dV33rMGHN8aDHRqOqPgB81mvWBiOQlLiTTmV577bUu2c9ll13GZZdd1iX7Msb0LC1eOhOR/xSRLHE8LiKbgeldEJsxxpjjQGvu0XxeVcuAS4F+wM04r5AxxhhjWtSaRFP/tNxM4Neq+kajecYYY0yzWpNoNonISpxE84KIBIFoYsMyxhhzvGhNopkPLALOV9VKIAXn8plJoOO1m4Cu7o8GnOdyRo8ezdixY8nNbbHJvzGmk7WYaFQ1CgwFviEiPwAmquqWhEd2gjteE01HtKc/mnqFhYWUlJS0O8kZY9qvNa+gWQycD/zenXWHiExU1XsSGll38vwi+PjNzt3mwNFwRfw2FdYfTef0R2OMSb7WXDqbCcxQ1V+p6q+Ay4ErExuWsf5oOq8/GhHh0ksv5bzzzuOxxx7rwL+KMaY9WttNQC/gE3c8O0GxdF/N1Dy6gvVH07H+aF5++WUGDx7Mvn37mDFjBmeeeSZTpkxp1XEyxnRcaxLNd4HXRaQQp1nzFKBDl81EpA/wDDAc5/U216jqoRjlIkD9Naudqprvzp8O/ACnYcImYL6qht1l04CHAT9wQFWndiTW7sD6o+lYfzSDBw8GnBra3Llz2bhxoyUaY7pQaxoDPAVMAP7qfi4E1nVwv4uANap6GrDGnY6lSlXHup/6JOMBngCuU9VRwAfAPHdZL+CnQL6qng18poNxJo31R9M5/dFUVFQ0rFNRUcHKlSut0zNjulirLp2p6h6goH5aRHYCwzqw3znANHf8CaAIp/uB1sgBalS1/qy0CqeG9ThwA/BXVd3pxr2vAzEmlfVH0zn90ezdu5e5c+cCToK84YYbGi4ZGmO6Rov90cRcSeRDVT2p3TsVOayqvRpNH1LV3jHKhYESIAwsVtXnxLmOsgO4SlWLReSHwHRVHS0i9ZfMzgaCwA9V9bdxYlgALAAYNmzYeR988MFRy61fkuSx/miM6ToajSLNXPpuTmv7o2ltY4CmWsxOIrIaiNUU6d427GeYqu4WkVOBtSLypqq+JyLXAQ+JSCqwEicRgfN9zgMuBtKBV0VkQ6Paz5EvoPoY8Bg4HZ+1ISZjjOnR6vbto2zZ3yhdupRen55Dn3nzErq/uIlGRH5M7IQiOK3QmqWqlzSz7b0iMkhV94jIICDmJS5V3e0O3xeRImAc8J6qvgpMdrd1KXC6u8ounAYAFUCFiKwDxgDHJJoThfVHY4wBiFZUUL56NaVLCqjYsAGiUdJGj8Y3YEDC991cjaa5R6g7+nh1Ac4N/MXucEnTAiLSG6hU1RoR6QtcBHzPXdZfVfe5NZqFwLfd1ZYAPxERH06LtPHAQx2MtUez/miMOXFpOEzFqxsoXVpA+arVaFUV/iFDyPniArJn55N66ildEkfcRKOqTyRwv4uBP4rIfGAnbuswEckFblXVW4CRwM9FJIrTOm6xqta/f+RuEZnlzn9UVde6MW8XkRXAFpwXf/5SVbcm8HsYY0y3oqrUbN9OacFSSv+2jMj+A3iyssiePZvsOfmkjxvX7nsy7dXeezQdoqoHce6jNJ1fDNzijr8CxLy+o6p3A3fHWfZ94PudFqwxxvQAdXv2ULp0GWVLC6h5513w+wlMnUJ2fj6BqVPxuM+cJUNSEo0xxpiOi4RClL/wAqUFS6ncuBFUSR83joH/8y2Cl12Gr/cxjXmTwhKNMcb0IFpXR+illyhbupTyNWvRmhr8Jw+j75dvJ3v2bFKGdeQRx8Ro8UKdiPwoxuf/RGROVwR4ojpeuwno6v5o/vnPfzJ27NiGT1ZWFg8//HC79m9MsqgqVVu28PH93+adqdPY9aXbqHjlVXpddRXDn36KEStW0O/227tlkoHW1WjSgDOBP7nTVwFvAfNFJE9V70xUcCey+kRz2223tXndhx9+mBtvvJGMjIwERJY8zz33HLNmzeKss85q9TpnnHEGJSUlAEQiEYYMGdLwpgBjurvaXbsoW7qU0iUF1O7YgaSkEJg+nez82QQmTUJSUpIdYqu0JtF8CufJ+/qXVj6K85DkDI688PK49sDGB/jHJ//o1G2e2edMFl4Q/6071h9N5/dHs2bNGkaMGMHJJ5/c8j+QMUkSKS2lbMULlBYUUOV2q5Fx/vnk3DKf4KWX4s3KSnKEbdeaRDMEyARK3elMYLCqRkSkJv5qpiMWL17M1q1bKSkpYeXKlfz5z39m48aNqCr5+fmsW7eO/fv3M3jwYP72t78Bzksws7OzefDBByksLGz2FS71/dE88MADzJ07t6E/mm3btjFv3jzy8/Mb+qNJS0vjnXfe4frrr6e4uJi5c+fyl7/8hUceeYQVK1a0uj+aLVu2cO655wJH90eTmZnJAw88wIMPPtjw9un6/mh++9vfcuedd7Js2TLy8/OZNWsWV199dcP26/ujWb58Offddx+rV68+6l1njT399NPNdmlgTLJEa2sJvfgiZQVLCRUVoXV1pIwYQb+vfIXsWVfiHzIk2SF2SGsSzfeAEvfJ/PpuAr4jIpnA6gTG1m00V/PoCtYfTcf6o6nffkFBAd/97ndbPDbGdAVVper11yktKKDs+RVES0vx5uTQ+4bryZqdT9rZZzXbRUZP0mKiUdXHRWQ5cAFOovl6/athiPMsi+lc1h9Nx/qjAXj++ec599xzGdAFr9swpjm1O3Y4D1MuXUrdhx8iaWkEL7mE7PzZZE6ciPiOv8bArX089Hycd4tNwnlppUkw64+mc/qjqffUU0/ZZTOTNOFDh/jk97/nX9dey3uXX8GBRx8l5aShDPrudzntpZcY8oPvE5gy5bhMMtCKGo2ILMZJNL93Z90hIhNVtUO9bJrmWX80ndMfDThJbNWqVfz85z9v97Ewpq2i1dWEioooXVJAaP16CIdJPeMM+t99N1mzrsR/AtWuW+yPRkS2AGNVNepOe4HXVfWcLoivS+Tm5mrT5zGsX5Lksf5oTE+l0SiVxcWUFhRQvuIFoqEQvv79yZo1i+w5+aSdcUayQ+xUnd0fTS/gE3c8u91RGWPMcajmvfcoXVJA6bKlhHfvwZORQfDSS8nOn03G+PGI15vsEJOqNYnmu8DrIlLIkVZndtmsh7D+aIxJjPD+/ZQtX07pkgKqt20Dr5fMiybS/ytfJXjxdDzH2QPTHdGaVmdPuU2bz8dJNAtpfSMCk2TWH40xnSdaWUn5mrWUFhRQ8corEImQdvbZDLjow/ryAAAfyUlEQVRnEVlXXokvCZd7e4JWXTpT1T04nZUBICI7ge75Uh1jjOlEGolQ+dprlC4poHzVKqKVlfgGDyJn/nyy82eT+qlPJTvEbq+9bemOj6eIjDEmjup//pPSJQWULVtGeN8+PIEAwZlXkJ2fT0Zubpd3HtaTtTfRNN9UrQUi0gd4BhgO7ACuUdVDMcpFOPI+tZ2qmu/Onw78AKe75k3AfFUNi0g28Duc2pYP+IGq/rojsRpjThx1e/dStmwZpUsKqHn7bfD5CEyeTPbX7yGQl5fUzsN6sriJRkR+TOyEIjit0DpiEbBGVReLyCJ3OtZ7XqpUdWyTuDzAE8DFqvq2iPwvMA94HLgd2Kaqs0WkH/BPEfm9qtZ2MF5jzHEqEqqgfNUqSguWULnhNafzsDFjGPDf3yBr5sxu03lYT9Zc3a8Yp7bQ9FMM/EcH9zsHJ1ngDj/dhnVzgBpVrX+MfBVO1wXgJMagOO8sCeA0yY7/XpJuzPqjOVZ7+qMB+OEPf8ioUaM4++yzrS8aA4CGw4TWreOj/7qLdyZNYs8991C36yP6fulLjFjxPMOfeZo+n/2sJZlOErdGo6pPNJ0nIgNV9eNO2O8At4EBqrpHRPrHKZcmIsU4yWKxqj4HHAD8IpKrqsXA1cBJbvmf4DRa2A0EgWvrHzTtaaw/mmO1pz+arVu38otf/IKNGzc2vEj0yiuv5LTTTktgpKY7UlWq39pGacESyv62nMjBg3iys8n+9Byy8+eQPm7scfMSy+6mrfdolgPntqagiKwGYr07/t427G+Yqu4WkVOBtSLypqq+JyLXAQ+JSCpO3zj1tZbLgBJgOjACWCUi61W1LEZ8C4AFAMNa6JXu4+98h5rtndsfTerIMxn49a/HXW790XROfzTbt29nwoQJDUl36tSpPPvss3zta19rw7+W6cnqPvqI0qXLKF26lNr33kP8fgLTppE9J5/MKVPw9JDOw3qytiaaVqd7Vb0k7kZE9orIILc2MwjYF2cbu93h++6zPOOA91T1VZyXfCIilwKnu6vcjFPzUeBdEfkXTu+gG2Ns+zHgMXBeQdPa79VVrD+azumPZtSoUdx7770cPHiQ9PR0li9fTm5ui2/MMD1cpKyMshdeoGxJAZXu5dX0885j4H33kXX5ZXiz7QUnXamtieYXnbTfApwb+Ivd4ZKmBUSkN1CpqjUi0he4CKdvHESkv6ruc2s0C4Fvu6vtBC4G1ovIAOAM4P2OBttczaMrWH807e+PZuTIkSxcuJAZM2YQCAQYM2YMvuP0DbknOq2tJfTSS85LLAsL0dpaUoYPp99/3kHW7NmkDB2a7BBPWG36i1PV9t2dPtZi4I8iMh8nOXwGQERygVtV9RZgJPBzEYniNFpYrKr1d4LvFpFZ7vxHVXWtO///gN+IyJu4bzFQ1QOdFHPSWH80HeuPZv78+cyfPx+Ar3/96wy1E85xQ1WpfuMNp/Ow5c8TOXwYb58+9LrmGucllqNG2X2XbiApTxyp6kFVvVhVT3OHn7jzi90kg6q+oqqjVXWMO3y80fp3q+pIVT1DVR9uNH+3ql7qlh+lqr/r+m/XOaw/ms7rj2bfPufK7M6dO/nrX/9q/dIcB2p37mT/Tx7hvcsvZ8d113P4L38lc+KFDP3Zo5z2YhEDv3Ev6aNHW5LpJuwaQjdl/dF0Xn80V111FQcPHsTv9/PII4/Q25qs9kjhQ4coX7GC0oKlVL3+OoiQccEF9F3wRYKXzsAbDCY7RBNHi/3RnAisP5ruxfqjMfWitbWECosoXVpA6MV1UFdH6mmfIis/n+xZs/APGpTsEE9ond0fjTHGdAmNRqnavNl5z9gLLxAtK8Pbry99PvtZsufkk3rmmXZJrIexRHOcs/5oTE9R8/6/nIcply6j7qOPkPR0gjMuITt/DpkTxiPWWrDHsn+5Zqhqj//lZP3RtJ5dRu564YMHKfvbckqXLqX6zTfB4yHzwgvp9593ELz4YjzufUHTs1miiSMtLY2DBw+Sk5PT45ONaZmqcvDgwaOacJvEiFZXU75mDWUFSwm99BJEIqSOHEn/hQvJunIm/v7x3khleipLNHEMHTqUXbt2sX///mSHYrpIWlqaPWOTIBqNUrlxI6UFSyl/4QWiFRX4Bg4k5/M3kzV7Nmmnn97yRkyPZYkmDr/fzymnnJLsMIzp0arffpuypUspXbqM8Mcf48nMJHjZZWTnzybj/PMRrzfZIZouYInGGNOp6vbto2zZ3yhdupSa7dvB6yUwaRLZX7vb6TwsPT3ZIZouZonGGNNh0YoKylevprRgKRWvvgrRKGmjRzPg618n68qZ+HJykh2iSSJLNMaYdtFwmIpXN1C6tIDyVavRqir8Q4aQ88UFZM+eTeqppyY7RNNNWKIxxrSaqlKzfTulBUsp/dsyIvsP4MnKInv2bLLzZ5N+7rlIMy9YNScmSzTGmBbV7dlD6bJllBUUUPPOu+D3E5g6hezZ+QSmTcXjvkXbmFgs0RhjYoqEQpS/8AKlBUup3LgRVEkfN46B3/omwcsvx2cvJzWtZInGGNNA6+oIvfwyZQUFlK9Zi9bU4D95GH1vv53s/NmktNDtuTGxWKIx5gSnqlS/+SalBUspW76cyCef4O3Vi15X/RvZ+fmkjRljb8cwHZK0RCMifYBngOHADuAaVT0Uo9ww4JfASYACM1V1h4icAjwN9AE2A59T1Vq3e+ffAucBB4FrVXVHwr+QMT1M7a5dzsOUSwqo3bEDSUkhMH062fmzCUyahKSkJDtEc5xIZo1mEbBGVReLyCJ3emGMcr8Fvq2qq0QkAETd+Q8AD6nq0yLyM2A+8Kg7PKSqnxKR69xy1yb6yxjTE0RKSylb8QKlBQVUbdoEQMb559Nn/ufJuuwyvFlZSY7QHI+SmWjmANPc8SeAIpokGhE5C/Cp6ioAVQ258wWYDtzQaP3/wUk0c9xxgD8DPxERUXs1rzlBRWtrqVi3jtIlBYSKitC6OlJOPZV+d95J9uxZ+IcMSXaI5jiXzEQzQFX3AKjqHhGJ9crW04HDIvJX4BRgNU7NpzdwWFXDbrldQP1fyxDgQ3e7YREpBXKAAwn7JsZ0M6pK1eslTv8uz68gWlqKNyeHXtdfR3b+HNLOPsvuu5guk9BEIyKrgYExFt3byk34gMnAOGAnzj2dm4CCGGXrayyx/nqOqc2IyAJgAcAwa0ljjhO1O3Y4D1MuXUrdhx8iaWkEL76Y7Dn5ZE6caJ2HmaRI6P86Vb0k3jIR2Ssig9zazCBgX4xiu4DXVfV9d53ngAnAr4BeIuJzazVDgd2N1jkJ2CUiPiAb+CRGbI8BjwHk5ubaZTXTY4UPHaJs+XJKCwqofmMLiJAxYTx9b7uN4IwZeAPWeZhJrmT+vCkA5gGL3eGSGGX+DvQWkX6quh/nvkyxqqqIFAJX47Q8a7x+/XZfdZevtfsz5ngTrakhVFjo3HdZvx7CYVJPP53+d99F1qxZ+AcMSHaIxjRIZqJZDPxRRObjXBb7DICI5AK3quotqhoRkbuANW4DgE3AL9z1FwJPi8j9wOvA4+78x4EnReRdnJrMdV32jYxJII1GqSwuprSggPIVLxANhfD170+ff/93sufkk3bGGckO0ZiYxH7sO5fOiouLkx2GMTHVvPcepUsKKF22lPDuPUhGBlkzZpA9J5+M8eOt8zCTNCKySVVzWypndwaN6WZUlZp//IPywkLKV6+mZtt28HjIvOgi+n/lqwQvno4nIyPZYRrTapZojOkGorW1VL62kVDhWsoLiwjv2QMipJ0zmgH3LCJr5kx8/folO0xj2sUSjTFJEv7kE0IvriNUWEjFSy8RraxE0tPJnDiR4JdvJzB1Kr6+fZMdpjEdZonGmC6iqtS+/z6hwkLK1xZSVVIC0Si+/v3Jmj2bQN40MidMwJOWluxQjelUlmiMSSCtq6Ny02YnuRQWUrdzJwCpZ42k7623Epg+3Z7SN8c9SzTGdLJIWRmh9esJrS0ktH490bIyxO8nY8IEcm6+icC0afgHDUp2mMZ0GUs0xnSC2p073VpLEZXFxRAO4+3Th+DFFxOYnkdg4kQ8mfaEvjkxWaIxph00EqHqjS1uK7FCat99D4CUT40g5+abCeTlkT7mHHvGxRgs0RjTatGKCkIvv0yosIhQURGRQ4fA5yMjN5fen/kMgbw86+rYmBgs0RjTjLo9ewgVFVG+tpDKDRvQujo8WVkEpkwhkDeNwOTJ1lmYMS2wRNMB7+8PsWb7PlJ8HlJ9HlL9HlJ9XlK8R8br5zvz3GmfhxSfM89aG3UvGo1S/da2hlZiNdu3A+AfNozeN9xAYPp0Ms4dh/j9SY7UmJ7DEk0HvLW7jG8v396hbRxJPN6jk1V98vI1Sli+oxNYSuPlDcms6XJLdi2JVldTsWGD00qsqIjwvn3g8ZA+diz97/ov55LYqaee8MfJmPayRNMBM0cPIu/M/tTURagJR6kJR6kNR6kJu9N1UWojEWrqou7yiLvc/dRFqIlEm11eXh3mYLj2yDab7KMz3ol6JIkdWxtrmsxSW1reNME1TppxkqjfK11+Eg8fOOBcEissouKVV9CqKjwZGWROmuS0Eps6FV/v3l0akzHHK0s0HeD1CIFUH4HU5BxGVaUuotRGom1KdkfKHL1evOXJTnZHJS9/7Jpe0+XHJDuvkLbrX/hfexl59SV0+1uginfgQIKf/jRZ0/PIHD8eT0pKx7+MMT2Aqjb87Xo8if2hZ4mmBxMRUnxCis+T9GR3TG2s8XRd09pa+5LdgXAttY2SXU1dxEmycZKdLxpm9IH3Gf/xNibseYtg1SEA/tHrJF4741I2DDqbf2UNgipBlpeR8sKahmR3bLJqOdkdnSSb1t6au7zZtTW7+hNMVJWoOzwy7czTOMP6Mseu32h5tPltNruNKG0q78yLEUPD+k2+YzROzPHKN43/mPiaL9/a43zMNmMcwzZtK9py+Xq3Th3BoivOTOj/OUs0pkMaJ7tkaZzsqg98QsX69dSse5Hoa69AZSWakkLdmFwO506kfOwFkNWHseEII1uR7JwaYfuTXVuIODW7xvfS6o/rUSeJaDMnqFachKwLKocIeETwiPP/2NMwLUctc6YbL3fLe5ovL9BQJub6Ah6Pp+X9eerLN17eivga9hGjPEfK5A5P/CViSzSmx6v91w63ldhaqja/DtEo3n596XXlTAJ508m8cAKe9PSExtBcze5IsmpaWzs2mcVKdkJrTkJtPEk2dxJqckJrdv2j9hfjpN3sSbKFk7yn5X02HbY+cWCNO7pQUhKNiPQBngGGAzuAa1T1UIxyw4BfAicBCsxU1R0icgrwNNAH2Ax8TlVrReSrwC1AGNgPfF5VP0j8NzJdScNhql5/nfK1hYQKC6ndsQOA1DPOIOeLCwjm5ZE2ahTi6bpaVneo2RnTXSWrRrMIWKOqi0VkkTu9MEa53wLfVtVVIhIAou78B4CHVPVpEfkZMB94FHgdyFXVShH5EvA94NpEfxmTeJHycipeeonywkIqXlxHpLQU/H4yL7iA3p+7keC0afiHDEl2mMaYGJKVaOYA09zxJ4AimiQaETkL8KnqKgBVDbnzBZgO3NBo/f8BHlXVwkab2ADcmJDoTZeo3fURocJCQoVrqfh7MdTV4e3Vi8C0aQTy8sicdBHeQCDZYRpjWpCsRDNAVfcAqOoeEekfo8zpwGER+StwCrAap+bTGzisqmG33C4g1k/Z+cDznR65SRiNRqnesoXywiJChYXUvP02ACmnnkqff/8cwenTSR871l5UaUwPk7BEIyKrgYExFt3byk34gMnAOGAnzj2dm4CCGGWPakcjIjcCucDUZuJbACwAGGYvQkyaaGUlFa++SvnatYReXEfkwAHwesk491z6L1xIMG8aKcOHJztMY0wHJCzRqOol8ZaJyF4RGeTWZgYB+2IU2wW8rqrvu+s8B0wAfgX0EhGfW6sZCuxutO1LcJLZVFWtaSa+x4DHAHJzc63BZxeq27uXUGER5YVrqXx1A1pbiycQIDBlMoG8POdFlb16JTtMY0wnSdalswJgHrDYHS6JUebvQG8R6aeq+3HuyxSrqopIIXA1TsuzhvVFZBzwc+ByVY2VvEwSqCo127c3tBKrfustAPxDh9LrumsJ5uWRcd55iD2Vb8xxSTQJT2+JSA7wR2AYzmWxz6jqJyKSC9yqqre45WYA/w8QYBOwwG3GfCpHmje/DtyoqjXu5brRwB53VztVNb+leHJzc7W4uLhzv+QJLlpTQ+Vrr1FeWEiosIjwxx+DCOljxhDIyyM4PY+UT33KnmUwpgcTkU2qmttiuWQkmu7GEk3nCH/yCaGiFwkVriX08itoZSWSnk5g0kUEpuURmDYVX05OssM0xnSS1iYaezOAaTdVpfbddxtaiVWVlIAqvgEDyM6fTXD6dDLGj8eTmprsUI0xSWSJxrSJ1tVRuWmT00qssIi6Dz8EIO2ss+h7++0E8qaRdtZZdknMGNPAEo1pUaS0lNC69c7Dk+vXEy0vR1JSyLhwAjnzP09g2jT8A2O1ZDfGGEs0Jo7aDz5oaCVWuWkTRCJ4c3IIXjqDYF4emRMn4snISHaYxpgewBKNAUAjEapKSpy3IK8tpPb99wFIPe00cubPJzg9j7RzzunSF1UaY44PlmhOYJFQBRUvv0xo7VpC69YROXQIfD4yzs+l93XXEZieR8rQockO0xjTw1miOcHU7d7d8GxL5WuvoXV1eLKzCUyZQnB6HpmTJuENBpMdpjHmOGKJ5jin0SjVb73V0Eqs5h//ACDl5JPpfeONBPKmkXHuuYjP/isYYxLDzi7HoWhVFRWvbnDutxQVEtl/ADwe0s8dR/+77yKQN53UU09JdpjGmBOEJZrjRHj/fsqLigitLaTi1VfR6mo8mZlkTp5MMG8amVOm4Oud+L7BjTGmKUs0PZSqUvPPfza0Eqt+800A/IMH0+uqqwhMzyPz/PPtRZXGmKSzRNODRGtrqdz4d0Jr11JeVEh4t/Pu0LRzzqHfnf9JIC+P1NNPt6fyjTHdiiWabi586BChF18kVFhExfr1RCsrkbQ0MidOJPClLxGYOhV//1gdlBpjTPdgiaabUVVq//Uvp9ZSWETV669DNIqvXz+yZs0ikDeNzAsvxJOWluxQjTGmVSzRdAMaDlO5abNzv6VwLXUf7AQgdeRI+t76RQJ500k7+yx7Kt8Y0yNZokmSSHk5FevXO+8TW7+eaGkp4veTMWECfebNIzhtGv7Bg5MdpjHGdFjSEo2I9AGeAYYDO4BrVPVQjHLDgF8CJwEKzFTVHSJyCkd62dwMfE5VaxutdzXwJ+B8Ve0WvZrVfvihW2sppPLvxRAO4+3dm2BentNKbOJFeAOZyQ7TGGM6VTJrNIuANaq6WEQWudMLY5T7LfBtVV0lIgEg6s5/AHhIVZ8WkZ8B84FHAUQkCNwBvJboL9EcjUSo2rKF0NpCQkWF1LzzLgApI0aQc/NNBPLySB8zBvF6kxmmMcYkVDITzRxgmjv+BFBEk0QjImcBPlVdBaCqIXe+ANOBGxqt/z+4iQb4P+B7wF2JCj6eaEUFoVdeIVRYRKioiMgnn4DXS0ZuLv0XXUUwL4+Uk0/u6rCMMSZpkploBqjqHgBV3SMisdrong4cFpG/AqcAq3FqPr2Bw6oadsvtAoYAiMg44CRVXSYiXZJo6j7++MglsQ2vobW1eIJBAlOmEMjLIzB5Et7s7K4IxRhjup2EJhoRWQ3E6nrx3lZuwgdMBsYBO3Hu6dwEFMQoqyLiAR5yy7QU2wJgAcCwYcNaGc7RKjZsYN/3vk/1tm0A+IcNo/f11xHIm07Geecifn+7tmuMMceThCYaVb0k3jIR2Ssig9zazCBgX4xiu4DXVfV9d53ngAnAr4BeIuJzazVDgd1AEBgFFLlPxw8ECkQkv2mDAFV9DHgMIDc3V9vz/Tzp6UhqKv3+66vOJbERI+ypfGOMaSKZl84KgHnAYne4JEaZvwO9RaSfqu7HuS9TrKoqIoXA1Tgtz+YBS1S1FOhbv7KIFAF3JarVWfqYMQx/6g+J2LQxxhw3kvkE4GJghoi8A8xwpxGRXBH5JYCqRnBu6K8RkTcBAX7hrr8Q+KqIvAvkAI93cfzGGGNaQVTbddXouJKbm6vFxd3iURtjjOkxRGSTqua2VM7eaWKMMSahLNEYY4xJKEs0xhhjEsoSjTHGmISyRGOMMSahLNEYY4xJKGveDIjIfuCDdq7eFzjQieF0lu4aF3Tf2CyutrG42uZ4jOtkVe3XUiFLNB0kIsWtaUfe1bprXNB9Y7O42sbiapsTOS67dGaMMSahLNEYY4xJKEs0HfdYsgOIo7vGBd03NourbSyutjlh47J7NMYYYxLKajTGGGMSyhJNK4nIr0Rkn4hsjbNcRORHIvKuiGwRkXO7SVzTRKRURErczze7IKaTRKRQRLaLyFsi8p8xynT58WplXMk4XmkislFE3nDjui9GmVQRecY9Xq+JyPBuEtdNIrK/0fG6JdFxNdq3V0ReF5FlMZZ1+fFqZVzJPF47RORNd7/HvK4+oX+TqmqfVnyAKcC5wNY4y2cCz+P0mTMBeK2bxDUNWNbFx2oQcK47HgTeBs5K9vFqZVzJOF4CBNxxP/AaMKFJmduAn7nj1wHPdJO4bgJ+0pXHq9G+vwr8Ida/VzKOVyvjSubx2gH0bWZ5wv4mrUbTSqq6DvikmSJzgN+qYwNOV9ODukFcXU5V96jqZne8HNgODGlSrMuPVyvj6nLuMQi5k3730/Tm6RzgCXf8z8DFkuB+w1sZV1KIyFDgSuCXcYp0+fFqZVzdWcL+Ji3RdJ4hwIeNpnfRDU5irgvdyx/Pi8jZXblj95LFOJxfw40l9Xg1Exck4Xi5l1tKgH3AKlWNe7xUNQyU4vQsm+y4AK5yL7X8WUROSnRMroeBrwHROMuTcrxaERck53iB8yNhpYhsEpEFMZYn7G/SEk3nifVrqTv8+tuM85qIMcCPgee6asciEgD+AtypqmVNF8dYpUuOVwtxJeV4qWpEVccCQ4ELRGRUkyJJOV6tiGspMFxVzwFWc6QWkTAiMgvYp6qbmisWY15Cj1cr4+ry49XIRap6LnAFcLuITGmyPGHHzBJN59kFNP51MhTYnaRYGqhqWf3lD1VdDvhFpG+i9ysifpyT+e9V9a8xiiTleLUUV7KOV6P9HwaKgMubLGo4XiLiA7Lpwkum8eJS1YOqWuNO/gI4rwvCuQjIF5EdwNPAdBH5XZMyyTheLcaVpONVv+/d7nAf8CxwQZMiCfubtETTeQqAf3dbbkwASlV1T7KDEpGB9demReQCnH/zgwnepwCPA9tV9cE4xbr8eLUmriQdr34i0ssdTwcuAf7RpFgBMM8dvxpYq+4d3GTG1eQafj7Ofa+EUtV7VHWoqg7HudG/VlVvbFKsy49Xa+JKxvFy95spIsH6ceBSoGlL1YT9Tfo6YyMnAhF5CqdFUl8R2QV8C+fmKKr6M2A5TquNd4FK4OZuEtfVwJdEJAxUAdcl+g8O55fd54A33ev7AF8HhjWKKxnHqzVxJeN4DQKeEBEvTmL7o6ouE5H/BYpVtQAnQT4pIu/i/DK/LsExtTauO0QkHwi7cd3UBXHF1A2OV2viStbxGgA86/6G8gF/UNUVInIrJP5v0t4MYIwxJqHs0pkxxpiEskRjjDEmoSzRGGOMSShLNMYYYxLKEo0xxpiEskRjjDEmoSzRGBOHiOSLyKJO3F6RiOTGmJ8rIj9yx28SkZ/EWT8Ua34nxLW8/sHMZsrEi32siMxMRFzm+GEPbBoTh/uAXUEX7KcYOKZ/kK6iqh1JFGOBXJyH/YyJyWo05oQkIsNF5B8i8ksR2SoivxeRS0TkZRF5R0QuaFy7EJHfiNMp1Csi8r6IXN3C9r8mTidTb4jI4kaLPiNOZ2Jvi8hkt+w0id1J1iki8qqI/F1E/q+F/f3UfeIcEXlWRH7ljs8Xkfvd8RvdfZeIyM/dJ/7rO8Tq647/t3tcVonIUyJyV7zYRSQF+F/gWneb1zZ/1M2JyhKNOZF9CvghcA5wJnADMAm4C+fVNE0NcpfPAhbHWA6AiFwBfBoY774F+nuNFvtU9QLgTpzXBTXnh8Cjqno+8HELZdcBk93xIcBZ7vgkYL2IjASuxXmD71ggAny2Sdy5wFU43Sf8G05NpbGjYlfVWuCbOJ2KjVXVZ1qI0ZygLNGYE9m/VPVNVY0CbwFr3PeavQkMj1H+OVWNquo2nHdHxXMJ8GtVrQRQ1cZvDa5/Y/SmOPto7CLgKXf8yRbKrgcmi8hZwDZgr/sCxwuBV4CLcd4U/Hf3PW8XA6c22cYkYImqVrkdwy1tsrwtsRvTwO7RmBNZTaPxaKPpKLH/NhqXb663RiF+Px7124jE2UdTrXoZoap+JCK9cV7jvw7oA1wDhFS13H0j9ROqek8LcTenrbEbA1iNxphEWAl8XkQyAET+f3t3rNJAEARg+J8uhTYpbO0Fax9C7OzsbW1S+iQ+iSAiRMUiQdDat9DGaixm1aB4CYmHqP/XHBy7t9sNOzvcxHDJ71zz/tfhg66BzQ2V1hpTJ5xRewKcA/sRsfG6p4jY/DD/CtiLiEFUc7jdBdZ8BNYXGKd/zEAjfbPMPKWq1aYtTTWaM+UrR1QnxAnVuGueS+oe5YHqFDps72jpvmOqle89cEbdOc3ue9L2fUelyaZUC+QuF8CWxQDqYpsASW8iYi0zn9ppbAwcZubtT+9Lv5t5VkmzTlpBwYC60zHIaGWeaKQlRcQ2n6vBnjNz5y+tKa3KQCNJ6pXFAJKkXhloJEm9MtBIknploJEk9cpAI0nq1QtlRRg6mximcwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch2_1.best_score_, gsearch2_1.best_params_))\n",
    "test_means = gsearch2_1.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch2_1.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch2_1.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch2_1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch2_1.cv_results_).to_csv('my_preds_maxdepth_min_child_weights_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(max_depth), len(min_child_weight))\n",
    "train_scores = np.array(train_means).reshape(len(max_depth), len(min_child_weight))\n",
    "\n",
    "for i, value in enumerate(max_depth):\n",
    "    pyplot.plot(min_child_weight, test_scores[i], label= 'test_max_depth:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'min_child_weight' )                                                                                                      \n",
    "pyplot.ylabel( '-Log Loss' )\n",
    "pyplot.savefig('max_depth_vs_min_child_weght_1.png' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': [4, 5, 6], 'min_child_weight': [1, 2, 3, 4, 5]}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#max_depth 建议3-10， min_child_weight=1／sqrt(ratio_rare_event) =5.5\n",
    "max_depth = [4,5,6]\n",
    "min_child_weight = [1,2,3,4,5]\n",
    "param_test2_2 = dict(max_depth=max_depth, min_child_weight=min_child_weight)\n",
    "param_test2_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.59275, std: 0.00306, params: {'max_depth': 4, 'min_child_weight': 1},\n",
       "  mean: -0.59280, std: 0.00358, params: {'max_depth': 4, 'min_child_weight': 2},\n",
       "  mean: -0.59290, std: 0.00359, params: {'max_depth': 4, 'min_child_weight': 3},\n",
       "  mean: -0.59287, std: 0.00333, params: {'max_depth': 4, 'min_child_weight': 4},\n",
       "  mean: -0.59288, std: 0.00346, params: {'max_depth': 4, 'min_child_weight': 5},\n",
       "  mean: -0.58854, std: 0.00444, params: {'max_depth': 5, 'min_child_weight': 1},\n",
       "  mean: -0.58916, std: 0.00315, params: {'max_depth': 5, 'min_child_weight': 2},\n",
       "  mean: -0.58931, std: 0.00395, params: {'max_depth': 5, 'min_child_weight': 3},\n",
       "  mean: -0.58954, std: 0.00396, params: {'max_depth': 5, 'min_child_weight': 4},\n",
       "  mean: -0.58907, std: 0.00362, params: {'max_depth': 5, 'min_child_weight': 5},\n",
       "  mean: -0.58918, std: 0.00328, params: {'max_depth': 6, 'min_child_weight': 1},\n",
       "  mean: -0.58929, std: 0.00390, params: {'max_depth': 6, 'min_child_weight': 2},\n",
       "  mean: -0.58951, std: 0.00358, params: {'max_depth': 6, 'min_child_weight': 3},\n",
       "  mean: -0.58843, std: 0.00262, params: {'max_depth': 6, 'min_child_weight': 4},\n",
       "  mean: -0.58855, std: 0.00302, params: {'max_depth': 6, 'min_child_weight': 5}],\n",
       " {'max_depth': 6, 'min_child_weight': 4},\n",
       " -0.5884293907889163)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb2_2 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=232,  #第一轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch2_2 = GridSearchCV(xgb2_2, param_grid = param_test2_2, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch2_2.fit(X_train , y_train)\n",
    "\n",
    "gsearch2_2.grid_scores_, gsearch2_2.best_params_,     gsearch2_2.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([163.62048912, 160.42671771, 165.53228559, 167.78677793,\n",
       "        167.70634294, 201.51547375, 203.67054029, 189.29326291,\n",
       "        190.75809865, 200.46543484, 232.09187174, 236.06308346,\n",
       "        234.02637997, 231.9012394 , 185.51336222]),\n",
       " 'std_fit_time': array([ 5.82005696,  2.96875681,  3.98194113,  1.88336185,  2.08158027,\n",
       "         1.51383768,  5.47129698,  5.18015306,  6.93610673, 18.7244339 ,\n",
       "         5.79275897, 13.10651969,  3.1166946 ,  5.43594296, 26.83281617]),\n",
       " 'mean_score_time': array([0.83199821, 0.82218924, 0.61518645, 0.63270798, 0.6406147 ,\n",
       "        0.8510169 , 0.76022983, 0.74211178, 0.80937734, 0.6933651 ,\n",
       "        0.882447  , 0.89666085, 0.87063584, 0.81918583, 0.65042453]),\n",
       " 'std_score_time': array([0.13433166, 0.14875823, 0.10392496, 0.06057141, 0.1354415 ,\n",
       "        0.07855427, 0.07703179, 0.05188343, 0.11049105, 0.02857867,\n",
       "        0.01183643, 0.02850478, 0.04075953, 0.05473986, 0.01530426]),\n",
       " 'param_max_depth': masked_array(data=[4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_min_child_weight': masked_array(data=[1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3, 4, 5],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'max_depth': 4, 'min_child_weight': 1},\n",
       "  {'max_depth': 4, 'min_child_weight': 2},\n",
       "  {'max_depth': 4, 'min_child_weight': 3},\n",
       "  {'max_depth': 4, 'min_child_weight': 4},\n",
       "  {'max_depth': 4, 'min_child_weight': 5},\n",
       "  {'max_depth': 5, 'min_child_weight': 1},\n",
       "  {'max_depth': 5, 'min_child_weight': 2},\n",
       "  {'max_depth': 5, 'min_child_weight': 3},\n",
       "  {'max_depth': 5, 'min_child_weight': 4},\n",
       "  {'max_depth': 5, 'min_child_weight': 5},\n",
       "  {'max_depth': 6, 'min_child_weight': 1},\n",
       "  {'max_depth': 6, 'min_child_weight': 2},\n",
       "  {'max_depth': 6, 'min_child_weight': 3},\n",
       "  {'max_depth': 6, 'min_child_weight': 4},\n",
       "  {'max_depth': 6, 'min_child_weight': 5}],\n",
       " 'split0_test_score': array([-0.58733201, -0.58736125, -0.58750293, -0.58779572, -0.58732911,\n",
       "        -0.58081778, -0.58392692, -0.58244275, -0.58301188, -0.58290437,\n",
       "        -0.58319092, -0.58321817, -0.58328488, -0.58409673, -0.58339847]),\n",
       " 'split1_test_score': array([-0.59250521, -0.59125172, -0.59171869, -0.59111989, -0.59125965,\n",
       "        -0.58661085, -0.58747944, -0.58743063, -0.58720278, -0.58787314,\n",
       "        -0.58994981, -0.58690106, -0.58858881, -0.58733472, -0.58840104]),\n",
       " 'split2_test_score': array([-0.59269694, -0.59186514, -0.59156244, -0.59244913, -0.59283708,\n",
       "        -0.59030368, -0.58992273, -0.59159449, -0.59089604, -0.58920255,\n",
       "        -0.58964332, -0.59219127, -0.59118181, -0.58951265, -0.58948839]),\n",
       " 'split3_test_score': array([-0.59626542, -0.59675827, -0.597571  , -0.59649633, -0.59695115,\n",
       "        -0.59171307, -0.59248065, -0.59181353, -0.59286139, -0.59242864,\n",
       "        -0.58985339, -0.58983951, -0.59046312, -0.58924584, -0.5886441 ]),\n",
       " 'split4_test_score': array([-0.59497526, -0.59676117, -0.59612842, -0.59651093, -0.59600573,\n",
       "        -0.59324813, -0.59197355, -0.5932821 , -0.59370927, -0.59291896,\n",
       "        -0.59327266, -0.59427848, -0.59405288, -0.59195808, -0.59282591]),\n",
       " 'mean_test_score': array([-0.59275483, -0.59279927, -0.5928965 , -0.59287418, -0.59287635,\n",
       "        -0.58853842, -0.58915649, -0.58931246, -0.58953602, -0.5890653 ,\n",
       "        -0.58918177, -0.5892854 , -0.58951402, -0.58842939, -0.58855132]),\n",
       " 'std_test_score': array([0.00305708, 0.00358355, 0.00359317, 0.00332849, 0.00345788,\n",
       "        0.00444295, 0.00315387, 0.00394855, 0.00395807, 0.00362103,\n",
       "        0.00328262, 0.00390321, 0.00357601, 0.00261764, 0.00302425]),\n",
       " 'rank_test_score': array([11, 12, 15, 13, 14,  2,  5,  8, 10,  4,  6,  7,  9,  1,  3]),\n",
       " 'split0_train_score': array([-0.54942536, -0.55060597, -0.55129691, -0.55231596, -0.55258638,\n",
       "        -0.51032136, -0.51451761, -0.51745076, -0.51862606, -0.52056737,\n",
       "        -0.46082428, -0.47036308, -0.47481379, -0.47863237, -0.48231542]),\n",
       " 'split1_train_score': array([-0.54718398, -0.54790167, -0.548918  , -0.54993746, -0.55119385,\n",
       "        -0.50977527, -0.51285352, -0.51565318, -0.51715714, -0.51785846,\n",
       "        -0.46173261, -0.46735261, -0.47297038, -0.47703086, -0.48055848]),\n",
       " 'split2_train_score': array([-0.54696307, -0.54798478, -0.5492468 , -0.54991491, -0.55035908,\n",
       "        -0.50912895, -0.51220793, -0.51451856, -0.51644699, -0.5178639 ,\n",
       "        -0.4620156 , -0.46711065, -0.47331486, -0.47634075, -0.47968936]),\n",
       " 'split3_train_score': array([-0.54603911, -0.54814989, -0.54875597, -0.54986988, -0.54997119,\n",
       "        -0.50983541, -0.51163992, -0.51509483, -0.51699103, -0.51810529,\n",
       "        -0.46102015, -0.46777694, -0.47271178, -0.47859939, -0.48038156]),\n",
       " 'split4_train_score': array([-0.54615266, -0.54779235, -0.54792452, -0.54932695, -0.55002642,\n",
       "        -0.50736342, -0.5096493 , -0.51297575, -0.51551143, -0.51644029,\n",
       "        -0.46120615, -0.4677338 , -0.47349013, -0.47784586, -0.48157804]),\n",
       " 'mean_train_score': array([-0.54715284, -0.54848693, -0.54922844, -0.55027303, -0.55082738,\n",
       "        -0.50928488, -0.51217366, -0.51513861, -0.51694653, -0.51816706,\n",
       "        -0.46135976, -0.46806742, -0.47346019, -0.47768984, -0.48090457]),\n",
       " 'std_train_score': array([0.00122001, 0.00106593, 0.00112238, 0.00104614, 0.00098207,\n",
       "        0.00103288, 0.00158838, 0.00146117, 0.00101747, 0.00133673,\n",
       "        0.00044605, 0.00117406, 0.0007286 , 0.00089379, 0.00092888])}"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch2_2.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.588429 using {'max_depth': 6, 'min_child_weight': 4}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZoAAAELCAYAAADgPECFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XmYHFW9//H3t2dNZsm+TxZWCSQxkYGwGEQwbCKLoIDLRVzwil7A+wsCFwGDKJGrVxAVxQ1EQUAFUVEDAYxiQkxiEkJASCCByZ5AMltm7e/vj6qZ6Zl0z/QsNT0z+byep5+uqq6uOl3J9KfPqVN1zN0RERGJSizTBRARkYFNQSMiIpFS0IiISKQUNCIiEikFjYiIREpBIyIikVLQiIhIpBQ0IiISKQWNiIhEKjvTBegLRo4c6VOmTMl0MURE+pUVK1bscvdRHa2noAGmTJnC8uXLM10MEZF+xcw2pbOems5ERCRSChoREYmUgkZERCKloBERkUgpaEREJFIKGhERiZSCRkREIqXraERkQGiMN/L0m09T01DDkLwhDM0bypC8IQzJHUJRbhFZsaxMF/GApaARkX7vpd0vMX/JfF7c/WLKdYpyixiSO6Q5hIrzipvnmx5D84ZSnFvcPF+cW0x2TF+T3aUjKCL9VnV9NXevvpv7193PkLwhfGPONzhq5FHsrd0bPOr2tkwnzJfXllNWWcae2j2U15bjeMp9FOYUtoRRQjA1BVJzzSl8vSnAcrJyevFI9G0KGhHpl/6++e/cuvRWNldu5oLDLuCLR3+RIXlDOr2duMepqKugvLa8OYj21O5pDqby2vLm6T21e9hatbV5Pu7xlNsdnD24OYTa1p7a1pwSX8vNyu3OYemTFDQi0q/s2reL2/95O396/U8cNOQg7j3jXo4ec3SXtxezWPOX/EQmpv2+uMepqq9KWmNKVpt6tfrV5tpUgzek3O6g7EFJa09Ja07h9ND8oeRl5XX5GERNQdMduzfAc3fAjItg0gkQUyc+kai4O4+uf5RvLf8W+xr2ccU7r+BT0z+VsRpAzGIU5RZRlFtESVFJ2u9zd6obqltqTW1qTm1rU6/tea25NtUQTx1Q+Vn5rcIn5bmoNuel8rPyMbOeOCQpKWi6Y/uLsPa3sPLnUFwCMz4UhM7oqZkumciA8tre17hlyS2s2L6Co8cczU3H38TBQw7OdLG6xMwoyCmgIKeACYUT0n6fu7OvYV+HNaemZZvKN7Gmdg17avdQH69Pud3LjrqM/y797574aCmZe+qTYAeK0tJS7/IwAXVV8O8/wZqHYP0i8EYYOz0InGkXQvG4ni2syAGkrrGOn7zwE370wo/Iz85nXuk8zjv0PGKm1oN0uTs1jTXNIVReV96qNjVt5DRmj5vdpW2b2Qp3L+1wPQVNN4MmUeVOePG3QehsXgEYHPyeIHSmfgDyirq/D5EDxIrtK5i/ZD6v732dMw86ky8d8yVGDhqZ6WJJAgVNJ/RY0CTatR5eeDgInbc3QvYgOOKsIHQOOQXU9VEkqb21e/n2im/zm1d/w4TCCXz5uC/z7gnvznSxJAkFTSdEEjRN3KHsn0HgrP0t7HsLBo+AaRcEoTPhaIj4RJxIf+Du/GXjX1iwbAF7avfw8SM/zufe+TkG5wzOdNEkBQVNJ0QaNIka6mDDoiB0/v0naKiB4QcHgTP9QzDikOjLINIHba7czK1Lb+Xvm//OUSOO4isnfIUjhh+R6WJJBxQ0ndBrQZOoZi+89PsgdF7/G+BQckwQOkedDwVqi5aBryHewC9f+iXfW/U9AK6cdSWXHHGJ7kvWTyhoOiEjQZNo72ZY+2tY8zBsXwuxbDj0fTDjw/COsyBnUObKJhKRF3e/yPx/zOelt17iPSXv4YbZNzCuUL00+xMFTSdkPGgSbVsbdiJ4BCq2QG4RHHlOEDpT5oB+6Uk/V11fzV3/uosHXn6AEfkjuO7Y65g7eW7kFw1Kz1PQdEKfCpom8UbY9FzQtLbucagth6JxMP3CoHltzDR1IpB+569v/pWvPf81tlZt5aJ3XMRV77qKolx1+++vFDSd0CeDJlH9Pnjlz0HT2qsLId4Ao48MajnTPwRD0r/9hUgm7KzeyYJlC1i4aSGHDj2Um4+/mZmjZ2a6WNJNCppO6PNBk6hqN6x7NAidN58HDKa8OwidqefAoKGZLqFIs7jH+fUrv+aOFXdQ21jLZ9/5WS476jLdQn+AUNB0Qr8KmkRvvQYv/BpW/wre2gBZefCOM4KmtUPnQvbAu9249B/r317PLUtv4V87/sWxY4/lpuNvYnLx5EwXS3qQgqYT+m3QNHGHLSth9UOw9jdQvQsGDQu6Sc+4CCbO1vkc6TW1jbXcs+Yefrr2pxTkFDCvdB7nHnKuTvYPQAqaTuj3QZOosR42PBN0Inj5j9CwD4ZODprWZlwEIw/LdAllAFu2dRm3LL2FTeWb+MDBH2DeMfMYnj8808WSiChoOmFABU2i2gp46Q/hRaF/BY/D+FnhnaUvgMLRmS6hDBB7avbwrRXf4rH1j1FSWMKNx9/ICeNPyHSxJGIKmk4YsEGTqGJbcD5nzUOwbQ1YVnBzzxkXBTf7zC3IdAmlH3J3/vj6H7l92e1U1FVw6VGX8tl3fpZB2brI+EDQp4PGzIYDDwFTgI3Ah9397STrNQIvhLNvuPs54fJTgf8FYkAl8Al3X29mk4GfAqOAt4CPuXtZR+U5IIIm0Y6Xgl5rLzwCe9+EnIJgGIMZH4KDToYsjYcnHXuz4k1uXXor/9jyD2aMnMFNx9/EO4a/I9PFkl7U14PmduAtd19gZtcBw9z92iTrVbp7YZLlrwDnuvtLZnYFcKy7f8LMHgH+4O73mdkpwGXu/vGOynPABU2TeBzeWBLUcl58DGr3QsHolotCx71TnQhkP/Xxen7+4s/5weofkBXL4qp3XcWHD/+w7k92AEo3aDL10/Vc4ORw+j7gWWC/oGmHA8Xh9BBgSzh9JPDFcPoZ4LHuFHLAi8VgyonB48zbg4tB1zwEy34ES78PI9/RclHoMHVLFXhh5wt8ZclXeOXtVzh10qlcd+x1jC0Ym+liSR+XqRrNHncfmjD/trsPS7JeA7AKaAAWuPtj4fI5BCGyDygHjnP3cjN7AHje3e80sw8CvwFGuvvu9spzwNZoUql+C9b9Lmhee+MfwbJJJwShc9R5QddpOaBU1lVy17/u4sGXH2TU4FH8z+z/4dRJp2a6WJJhGW86M7OngGQ/dW4A7kszaMa7+xYzOxh4GjjV3TeY2W+Bb7j782Z2DfAOd/+0mY0HvgscBCwGLgCOcve9SbZ9OXA5wKRJk47etGlTtz/zgPT2puBczpqHYNcrkJULh50WNK0ddhrk5Ge6hBKxRW8s4uvPf52d1Tu5+IiLuXLWlRTm7teiLQegjAdNuzs1+zdwsrtvNbNxwLPu3u5ZRDO7F/gD8FdgqbsfEi6fBPzZ3Y9ss34h8LK7d3gjMNVo0uAOW1e3dCKo2gH5Q+DI84LQmXR80BQnA8b2qu3ctuw2Fr2xiMOGHcZXjv8KM0bNyHSxpKc0NsDmFTB4eJevr+vr52geBy4FFoTPv2u7gpkNA6rdvdbMRgInArcDbwNDzOxwd38FmAu8FL5nJEEngzhwPUEPNOkJZjB+ZvCYe0twXc6ah4Mu0yvvgyETg3M5My6C0RoZsT9rjDfy8CsPc+fKO2mIN3D1u67mP476D3Jiuj9Zv/fW67Dh6eDx+uLgrvCz/xPO/Eaku81UjWYE8DAwCXgD+JC7v2VmpcB/hs1gJwA/BOIE3ZjvcPefhO8/H7glfO1t4JPu/pqZXQjcRtBZYDHweXev7ag8qtF0Q10VvPxE0LS24WnwRhg7Ixye+kIo0oni/uSVt19h/pL5rNm5huPHHc+Nx93IxOKJmS6WdFVNOWz8W/C3uX4RvP16sLy4BA49JbiW7qD3BLWaLujTTWd9jYKmh1TugLW/DUJny0qwWPCfeMZFMPVsyNO4I31VTUMNP1zzQ+5dey9FuUVcc8w1nH3w2bo/WX8Tb4Qt/2qptby5LPjxl1MAB80JguWQU2DEoT1y6YKCphMUNBHY9WrQtLbmIdizCbIHwRHvD259M+LQ4PY3+UN0nU4fsGTLEr669Ku8WfEm5x5yLvNK5zE0X8NN9Bt73mgJlteehZq9gAXXwR16ahAsJcdGcjd3BU0ndDVoquur2VO7h6LcIgpyCoiZTobvxz34VbXmIXjxt7Av4QYQWbnBBaKFo8Ln8NG0rHBMy3T+UIVSD3ur5i2++c9v8vvXfs/k4snceNyNzB43O9PFko7UVsDGv7eEy+71wfKi8WGN5b1w8MlQMDLyoihoOqGrQbNw40L+31//HwCGUZhTSGFuIUW5RRTmBM9tp4tyi4J1clqmi3OLKcwpJC8rb2A3VTTUQdkyKN8KlduDnmuVO8Pn7eH0zqCq31ZWLhSMah1EBaODMEoMqoJRwXU+A/k4dpO78/iGx/nm8m9SWV/JJ6d9kstnXE5eVl6miybJxBuDHp8bng7uzP7m8xCvD1oJpry7pTls1Dt6/f+9gqYTuho0ZRVlLNu2jIq6iuZHZX1lyum4x9vdXk4sp1UwJYZQewHVFGAFOQVkx/r5fcri8aDWs18QhY+m6aqdwXN7odQUTM3hNDph2ZgDMpQ2lW/iq0u+yvPbnmfmqJncfPzNHDrs0EwXS9raWxaESlNz2L63guVjZ7QEy6TjIDuzPw76evfmAaGkqISSog4v0wGCX5HVDdVJQ2i/UKqrpLy+nMq6SnZW76SyrpKK+gr2NezrcD+Dswd3OqASpwdlD8psrSoWg4IRwYMj21+3KZTaC6KKbbDthWA+3pBkfzlJAilZU97ofh1K9Y313Pvivfxg9Q/IzcrlxuNu5MLDL1Rzb19RVwUbn2tpDtv172B54Rg4/HQ45NSgOaxwVCZL2WUKml5iZhTkFFCQU9Dle0PVx+upqqsKAqm+JZQSp8vryptDq7Kukt01u9lYvjFYr66CBk/yZZsgy7L2C6ainISAavtabtF+6/baePCJoTR6avvrxuNQsydsoksIosSQqtwO29YGy9oNpRRBlFhbyh/aZy5gXbVjFfOXzGf9nvXMnTyX6469jtGDNRZRRsXjsP2FlmB5Yyk01kF2Pkw+Ad718aDWMvrIfvvjJpGCph/JieUwNH9ol3sEuTs1jTUdBlRiDauyrpI3Kt5onq+qr+pwP/lZ+clDKLeI4txixheMb64Nji8Y3zvBFIsF1woMHt6JUNqRpLaUcE5p+4thTak+yf6yE5rvxrQOorYhNWhYJKFUUVfBnSvv5OF/P8yYgjHcdcpdnDzx5B7fj6SpfCu89kzLuZbqXcHyMdNg9mfD5rDjIWfgjeWjoDmAmBmDsgcxKHtQl3/RNsYbqWqoah1QdZVU1rdMNwVV03x5XTmbKzdTUVfB3rq9NCTUFgxjbMHYIHgKS1o/F5UwLG9Y7zflJYYSHdzlwD08p5Sk2S5x2Y51wXOHodSm113hGBhSAsOmBNNpHAt356k3nuK2529jd81uPjr1o/zXrP9icM7gLh0O6aK66uCmtE3nWnasC5YXjApC5dCwOewAuKhZQSOdkhXLoji3mOLc4o5XTiLucXZW76SssoyyirKW54oy/rb5b+zat6vV+oOzB+8XPk3TEwonkJvV89cGdIpZ50MpWbNdU22pcnswMF2yUMrOh6GTgtAZOjkYumHo5GB+2GTIH8K2qm18benXeLbsWY4YfgR3nXIXR408KqIPL624w/a1Lc1hm5ZAY23QOWXS8fC++UHAjJnWZ5pVe4t6naELNvuS6vpqtlRuSRpEZZVl1Da23FHIMMYUjElaEyopLGF4/vD+213cPWi+q9gejIL69sbgsWdTOP1GMFBdqBF4cPho7irOxy3G54um8tGSU8gefjAMnQJDJ2a8h9KAVLE96BXWFC5VO4Llo6a29A6bfALkDszapLo3d4KCpn+Ie5zd+3a3Cp83K95snt+5b2er9QdlD0rZJDehcEL/v25k39vw9kZe3ryU+RseYW3tTk5kMDdW1DPh7TeDk8vNDIrHJ68JDZ0MReMOuF/ZXVJfE4xK23SeZXs40vzgEXDwe1sumCwen9ly9hJ1b5YBJ2YxRg0exajBo5g1etZ+r+9r2BfUhtrUhN6seJMlW5ZQ01jTav3Rg0fvVwuaWDSRkqISRuSP6PO1oersPH6w5Rl+/tLPGZI3hNtPup0zppwRlDseh4qtYQ1oU0JNaFNw197yLQT3ng1l5QbNck1BlNg8N2zKgTvYnXvQlNncHPYcNNQEPRAnHQen3hR0PR47Q0HdDtVoUI3mQODu7K7ZvV8tqCmUdlTvaLX+oOxBTCic0KomNLFoIiWFJYwvHE9+dmYHfPv75r9z69Jb2Vy5mQsOu4AvHv1FhuQNSX8DDbWw503YszEIn+ZmuXC6Zk/r9fOGwLBJCTWhKQm1o0kDq6dU1a6WE/gbnobKbcHykYcnNIedCHka/E1NZ52goJHaxlo2V25uFT6Jz20vlh09aPR+nROapkcOGhlZbWjXvl3c/s/b+dPrf2JK8RRuPv5mSsd2+HfeeTV7968JNU3veSP4VZ+ocOz+NaGmUCoeD7Gsni9jT2moDW7r0hQsW1cHy/OHBs1gh5wSNIsN1XAJbSloOkFBI+1xd96qeas5dN6seLNVCO2o3oEnNEPlZ+W31IbanCOaUDSBQdmd//Xv7jy6/lG+tfxb7GvYx2emf4ZPTf9UZnrdxePBSe9kNaE9m6B8MyTebimW09JFuzmAmkJpStBjrzebKd2DYcmbgmXj36G+OuhmPnF2S7iMm9m3A7IPUNB0goJGuqO2sTbpuaGm6eqG6lbrjxo0KmWX7ZGDRu53W5jX977OLUtuYfn25Rw95mhuOv4mDh5ycG9+xM5pqIPysjY1oYQgqt7dev3cwtRdtodO7pkeW9Vvtb5YsnxzsHz4IS3NYQfN0ZhJnaSg6QQFjUTF3Xm79u2UTXLbqra1qg3lZeW1OjcUsxgP/fsh8rPzmVc6j/MOPa//35+stqJ1ALVtnqtvHcwUjE5SEwqni0sgK0mfpoY6KPsnbFgUhMuWVYAH55oOfk9L77BhU6L/vAOYgqYTFDSSKXWNdWyt2rpfEDV1WKiqr+LMKWfypWO/xMhB0Y8vknHuwQWtrQJoY0so7S1rfcduywqb5cIgKh4PW9cEwxfXVQavlxzTUmsZPyt5MEmXqHuzSD+Qm5XL5OLJTC6evN9rTfem68o5nX7LrOU+cBOP2f/1xoawWS5JTeiVvwTnjoZOhhkfDpvDTgpGcpWMUtCI9FFN96aTBFnZLd2rk2mo1R0Q+qB+3tgrIpJAIdMnKWhERCRSChoREYmUgkZERCKloBERkUgpaEREJFIKGhERiZSCRkREIqWgERGRSCloREQkUgoaERGJlIJGREQipaAREZFIKWhERCRSChoREYlURoLGzIab2ZNm9mr4PCzFeo1mtip8PJ6w/BQzW2lma83sPjPLDpebmX3HzNab2Roze1dvfSYREUkuUzWa64BF7n4YsCicT2afu88MH+cAmFkMuA+42N2nAZuAS8P1zwQOCx+XA3dH+BlERCQNmQqacwnCgvD5vE68dwRQ6+6vhPNPAhckbPfnHlgKDDWzcT1RYBER6ZpMBc0Yd98KED6PTrFevpktN7OlZtYURruAHDMrDecvBCaG0xOANxPeXxYuExGRDMmOasNm9hQwNslLN3RiM5PcfYuZHQw8bWYvuPsGM7sY+LaZ5QELgYam3SbZhqco3+UEzWtMmjSpE0USEZHOiCxo3P19qV4zs+1mNs7dt4ZNWztSbGNL+PyamT0LzAI2uPsSYE64rdOAw8O3lNFSuwEoAbak2PY9wD0ApaWlScNIRES6L1NNZ4/TcgL/UuB3bVcws2FhjQUzGwmcCKwL50eHz3nAtcAPErb7H2Hvs+OAvU1NdCIikhmZCpoFwFwzexWYG85jZqVm9uNwnanAcjNbDTwDLHD3deFr15jZS8Aa4Pfu/nS4/AngNWA98CPgil75NCIikpK5q9WotLTUly9fnuliiPQb9fX1lJWVUVNTk+miSC/Iz8+npKSEnJycVsvNbIW7l6Z4W7PIztGIyMBVVlZGUVERU6ZMwSxZHxwZKNyd3bt3U1ZWxkEHHdSlbegWNCLSaTU1NYwYMUIhcwAwM0aMGNGt2mungsbMYmZW3OW9iciAoZA5cHT337rDoDGzB8ys2MwKCHp9/dvMrunWXkVE5ICRTo3mSHcvJ7hNzBPAJODjkZZKRKQde/bs4fvf/36X3nvHHXdQXV3dwyXqGSeffDJd7Zj02GOPsW7duub5zmyrsbGRWbNmcfbZZ3dp3x1JJ2hyzCyHIGh+5+71pLjaXkSkNwzUoOmOtkHTGXfeeSdTp07t4RK1SKfX2Q+BjcBqYLGZTQbKIyuRiPQr83//Iuu29OxXwpHji7n5A0elfP26665jw4YNzJw5k7lz5zJ69GgefvhhamtrOf/885k/fz5VVVV8+MMfpqysjMbGRm688Ua2b9/Oli1beO9738vIkSN55plnkm6/sLCQz3/+8zz11FMMGzaMr3/963zpS1/ijTfe4I477uCcc85h48aNfPzjH6eqqgqA7373u5xwwgk8+uijfO973+PJJ59k27ZtvOc972Hx4sWMHbv/Hbn27dvHZZddxrp165g6dSr79u1rfm3hwoXcfPPN1NbWcsghh/Czn/2MwsJCpkyZwkUXXdRc9gceeIAdO3bw+OOP89e//pVbb72V3/zmNwA88sgjXHHFFezZs4ef/OQnzJkzZ78ylJWV8cc//pEbbriB//u//0v/H6kTOqzRuPt33H2Cu58V3hV5E/DeSEojIpKGBQsWcMghh7Bq1Srmzp3Lq6++yrJly1i1ahUrVqxg8eLF/PnPf2b8+PGsXr2atWvXcsYZZ3DllVcyfvx4nnnmmZQhA1BVVcXJJ5/MihUrKCoq4stf/jJPPvkkjz76KDfddBMAo0eP5sknn2TlypU89NBDXHnllQCcf/75jB07lu9973t85jOfYf78+UlDBuDuu+9m8ODBrFmzhhtuuIEVK1YAsGvXLm699VaeeuopVq5cSWlpaasQKC4uZtmyZXzhC1/g6quv5oQTTuCcc87hf//3f1m1ahWHHHIIAA0NDSxbtow77riD+fPnA7BlyxbOOuus5m1dffXV3H777cRi0XVC7rBGY2ZXAT8DKoAfE9xv7DqCm1mKyAGuvZpHb1i4cCELFy5k1qxZAFRWVvLqq68yZ84c5s2bx7XXXsvZZ5+d9Nd8Krm5uZxxxhkATJ8+nby8PHJycpg+fTobN24EgotWv/CFL7Bq1SqysrJ45ZVXmt9/1113MW3aNI477jguueSSlPtZvHhxc0DNmDGDGTNmALB06VLWrVvHiSeeCEBdXR3HH3988/uatnnJJZfwxS9+MeX2P/jBDwJw9NFHN5d7/PjxPPHEEwD84Q9/YPTo0Rx99NE8++yz6R6eTkun6eyT7n6nmZ0OjAIuIwgeBY2IZJy7c/311/PZz352v9dWrFjBE088wfXXX89pp53WXBvpSE5OTnOX3lgsRl5eXvN0Q0Nws/hvf/vbjBkzhtWrVxOPx8nPz29+/+bNm4nFYmzfvp14PN5ubSFZ12F3Z+7cuTz44IMdvqe9rsdN5c7Kymoud6LnnnuOxx9/nCeeeIKamhrKy8v52Mc+xi9+8YuU2+yKdOpKTZ/iLOBn7r6a5LfjFxHpFUVFRVRUVABw+umn89Of/pTKykog+JLfsWMHW7ZsYfDgwXzsYx9j3rx5rFy5cr/3dsfevXsZN24csViM+++/n8bGRiBorrrssst44IEHmDp1arvnPU466SR++ctfArB27VrWrFkDwHHHHcdzzz3H+vXrAaiurm5VY3rooYean5tqOl35XLfddhtlZWVs3LiRX/3qV5xyyik9HjKQXo1mhZktBA4CrjezIiDe4yUREUnTiBEjOPHEE5k2bRpnnnkmH/nIR5q/cAsLC/nFL37B+vXrueaaa4jFYuTk5HD33cHI7pdffjlnnnkm48aNa/c8TUeuuOIKLrjgAh555BHe+973UlBQAMDXv/515syZw5w5c5g5cybHHHMM73//+5P26vrc5z7HZZddxowZM5g5cybHHnssAKNGjeLee+/lkksuoba2FoBbb72Vww8PRkSpra1l9uzZxOPx5lrPxRdfzGc+8xm+853v8Otf/zplubds2cKnP/3p5uaz3tDhTTXNLAbMBF5z9z1mNgKY4O5reqOAvUE31RTpnJdeeinS7rCS2pQpU1i+fDkjR47s1f0m+zfvsZtqunvczEqAj4RtgX919993tbAiInJgSafX2QLgGOCX4aIrzewEd78+0pKJiERs9uzZzU1TTe6//36mT5/eo/v5y1/+wrXXXttq2UEHHcSjjz7a6W019R7rT9I5R3MWMNPd4wBmdh/wL0BBIyL92vPPP98r+zn99NM5/fTTe2VffVG6V+gMTZgeEkVBRERkYEqnRnMb8C8ze4agW/NJqDYjIiJpSqczwINm9izBeRoDrkUDpomISJrSGsrZ3bcCjzfNm9kbBMMFiIiItKurNRPdGUBEMmagDhOQifFopkyZwvTp05k5cyalpR1eEtMlXQ0ajUcjIhkzUIOmO7ozHs0zzzzDqlWruhxyHUnZdGZmd5E8UIzWvdBE5ED2p+tg2ws9u82x0+HMBSlf1ng0PTceTW9or0azHFiR5LEc+K/oiyYikpzGo+m58WjMjNNOO42jjz6ae+65pxv/KqmlrNG4+32R7FFEBpZ2ah69QePRdH08GgiGChg/fjw7duxg7ty5HHHEEZx00klpHad0pdXrTESkr9J4NF0fjwaC4IGghnb++eezbNmyHg8aXQ8jIv2OxqPpmfFoqqqqmt9TVVXFwoULmTZtWqe2kQ7VaESk39F4ND0zHs327ds5//zzgSAgP/KRjzQ3GfakdMaj+U6SxXuB5e7+ux4vUQZoPBqRztF4NJnTH8ejSafpLJ9g4LNXw8cMYDjwKTO7o/PFFRHzbbGDAAAUUElEQVSRA0k6TWeHAqe4ewOAmd0NLATmAj3ceV5EpPdoPJrekU7QTAAKCJrLCKfHu3ujmdWmfpuISN+m8Wh6RzpBczuwKryDc9MwAV83swLgqQjLJiIiA0A6wwT8xMyeAI4lCJr/cfct4cvXRFk4ERHp/9Lt3nwM0HRZbSOwpZ11RUREmnXY68zMFgBXAevCx5Vmdlt3dmpmw83sSTN7NXwelmK9RjNbFT4Sx8M5xcxWmtlaM7vPzLLD5UeY2RIzqzWzed0po4iI9Ix0ujefBcx195+6+0+BM4D3d3O/1wGL3P0wYFE4n8w+d58ZPs4BMLMYcB9wsbtPAzYBl4brvwVcCXyzm+UTkT5soA4TkInxaPbs2cOFF17IEUccwdSpU1myZEmX9t+edG9BkzgswJAe2O+5BGFB+HxeJ947Aqh196b7MTwJXADg7jvc/Z9AfQ+UUUT6qIEaNN3R1fForrrqKs444wxefvllVq9eHcmFuOmco7kN+JeZPUNLr7Pru7nfMeHw0Lj7VjMbnWK9fDNbDjQAC9z9MWAXkGNmpe6+HLgQmNjN8ohIF31j2Td4+a2Xe3SbRww/gmuPvTbl6xqPpmfGoykvL2fx4sXce++9QHDX6tzc3PT/odLUYY3G3R8EjgN+Gz6OBxZ39D4zeyo8h9L2cW4nyjcpvL3BR4A7zOwQD+6ZczHwbTNbBlQQBFGnmNnlZrbczJbv3Lmzs28XkQzSeDQ9Mx7Na6+9xqhRo7jsssuYNWsWn/70p5uDsyel1essrH0knox/A5jUwXvel+o1M9tuZuPC2sw4YEeKbWwJn18Lr+OZBWxw9yWEveDM7DTg8HQ+R5tt3wPcA8G9zjr7fhEJtFfz6A0aj6br49E0NDSwcuVK7rrrLmbPns1VV13FggUL+OpXv5r2sUpHV+/enHoAhPQ8TnACf0H4vN/NOcOeaNXuXmtmI4ETCS4excxGu/sOM8sDrgW+1s3yiEg/pfFouj4eTUlJCSUlJcyePRuACy+8kAULen4gu66OR9PdGsACYK6ZvUpwz7QFAGZWamY/DteZCiw3s9XAMwTnaJrOdF1jZi8Ba4Dfu/vT4fvHmlkZ8N/Al82szMyKu1lWEeljNB5Nz4xHM3bsWCZOnMi///1vABYtWsSRRx7ZqW2kI2WNxszuInmgGK17oXWau+8GTk2yfDnw6XD6H0DSO9u5+zUkuSuBu28DSrpTNhHp+zQeTc+MRwNBM99HP/pR6urqOPjgg/nZz37W5WOSSsrxaMzs0qQvhNz9vvZe7080Ho1I52g8mszpj+PRpKzRJAsSMxsb1hpERETS0tnOAE8A74qiICIivU3j0fSOzgZNd3ubiYj0GRqPpnd0ttfZjyIphYj0O6nO78rA091/604Fjbt37eZCIjKg5Ofns3v3boXNAcDd2b17d6vrhDqrqxdsisgBrKSkhLKyMnT7pgNDfn4+JSVdv3JEQSMinZaTk8NBBx2U6WJIP9HVOwOIiIikRUEjIiKRUtCIiEikFDQiIhIpBY2IiERKQSMiIpFS0IiISKQUNCIiEikFjYiIREpBIyIikVLQiIhIpBQ0IiISKQWNiIhESkEjIiKRUtCIiEikFDQiIhIpBY2IiERKQSMiIpFS0IiISKQUNCIiEikFjYiIREpBIyIikVLQiIhIpBQ0IiISKQWNiIhESkEjIiKRUtCIiEikFDQiIhKpjASNmQ03syfN7NXweViK9RrNbFX4eDxh+SlmttLM1prZfWaWHS7/qJmtCR//MLN39tZnEhGR5DJVo7kOWOTuhwGLwvlk9rn7zPBxDoCZxYD7gIvdfRqwCbg0XP914D3uPgP4KnBPlB9CREQ6lqmgOZcgLAifz+vEe0cAte7+Sjj/JHABgLv/w93fDpcvBUp6oKwiItINmQqaMe6+FSB8Hp1ivXwzW25mS82sKYx2ATlmVhrOXwhMTPLeTwF/SlUAM7s83PbynTt3du1TiIhIh7Kj2rCZPQWMTfLSDZ3YzCR332JmBwNPm9kL7r7BzC4Gvm1mecBCoKHNvt9LEDTvTrVhd7+HsGmttLTUO1EmERHphMiCxt3fl+o1M9tuZuPcfauZjQN2pNjGlvD5NTN7FpgFbHD3JcCccFunAYcnbHsG8GPgTHff3VOfR0REuiZTTWeP03IC/1Lgd21XMLNhYY0FMxsJnAisC+dHh895wLXAD8L5ScBvgY8nnMMREZEMylTQLADmmtmrwNxwHjMrNbMfh+tMBZab2WrgGWCBu68LX7vGzF4C1gC/d/enw+U3EXQW+H7YJXp5L30eERFJwdx1eqK0tNSXL1cmiYh0hpmtcPfSjtbTnQFERCRSChoREYmUgkZERCKloBERkUgpaEREJFIKGhERiZSCRkREIqWgERGRSCloREQkUgoaERGJlIJGREQipaAREZFIKWhERCRSChoREYmUgkZERCKloBERkUgpaEREJFIKGhERiZSCRkREIqWgERGRSCloREQkUgoaERGJlIJGREQipaAREZFIKWhERCRSChoREYmUgqYb4nEnHvdMF0NEpE/LznQB+rM/v7iNK365kqyYkZsVIyfLyM3OIi+7aTpGTlaM3OwYuW2ecxKe87Kbpo3crCxysoPt5bVZLzc7yTbabj9h37lZMcws04dJRA5wCppuOHR0IV983+HUNTZS1xCnvtGpbYiH0wnPjXFqG+JU1ja0eq2uIU5do1PX0Eh9o1PXGKexh2tIQXjFyAmDp3WwJQZTVhhWtl8YJoZZTsJz3n6hGQTcfgGZEJp5OTEG5WQpAEUOIAqabjh8TBGHjynq0W02xp36MJjaBlZtQxBa9U3P4Wu1Yci1CrE2z/stTwjB+gZn7776/QOyIeE9jXG8hzIwZlCYl01Rfg5F+dnhdDaF4XxR03y4TmF+dri8ZbowL5v8nKyeKZCIREpB08dkxYysWFaf+xJ1dxrCENw/sHy/cKtPEXa1DXEqaxqorG2gvKa+eXpXZR0bd1dTUVNPRU0DtQ3xDsuUmxVrFTzBcw7F+dkJy8PwajNfmJdNcX4OBXlZZGfpVKVIlBQ0khYzIyfLyMmKMTg3+v3VNjRSVdvYHDwVYSBV1NSHz02PlvnKmgbK3q5uma9tSKspcnBuVqtaVXGb4GobVC01rJYaV0GumgNFUlHQSJ+Ul51FXnYWwwu6nmruzr76RiprGigPg6cyDKeKhHBqFV5hmG3bW9McVpW1DR3uK2ZQENaSWkIrDKK87Nbh1aqJsHebA92duENDPE48Do3uNDZ68BwPHx70pmwI5+PuNDQGz43h8qbp5vUS3hOPt9lee9ttWj8OjfF4+L5wOk64TjjdZt+J221M2GYqiT8ErNXyNut1+j2WdHmq9S3V+inWoZvl62j9U6aO5uwZ44mSgkYGLDNjcG42g3OzGV3c9e00xp2qutbBlDKoahqorA1qYW9V1bFpd3VzzasrzYEFecGfaDpf3u2vE4ZLH++NHzQdG1lmLdMxI2ZGdtN0DLJjMWLWtH6MrBhkmRGLtXxFJ37UxPxpdQjaBFPq93jy5Sm26ykCL61ttrMdTzGT6j3pfJ7Devg8czIKGpEOZMWM4vwcivNzurWdurDnYVDDSmjyq61PaApsmW86fwWQlxMjFn75ZodfvMGXbvgFbC3TsSRf1Flt1kn88g7Wh6ysWPg+mr+8g3USprNa9p0YCLE25epqOWRgykjQmNlw4CFgCrAR+LC7v51kvUbghXD2DXc/J1x+CvBNIBdYAXzK3RvM7Fzgq0AcaACudve/R/tpRNKTmx1jeHZut5oDRfqjTHW3uQ5Y5O6HAYvC+WT2ufvM8NEUMjHgPuBid58GbAIuDddfBLzT3WcCnwR+HOWHEBGRjmUqaM4lCAvC5/M68d4RQK27vxLOPwlcAODuld7SQFlAmyZNERHpfZkKmjHuvhUgfB6dYr18M1tuZkvNrCmMdgE5ZlYazl8ITGx6g5mdb2YvA38kqNWIiEgGRXaOxsyeAsYmeemGTmxmkrtvMbODgafN7AV332BmFwPfNrM8YCHB+RgA3P1R4FEzO4ngfM37UpTvcuBygEmTJnWiSCIi0hmRBY27J/2CBzCz7WY2zt23mtk4YEeKbWwJn18zs2eBWcAGd18CzAm3dRpweJL3LjazQ8xspLvvSvL6PcA9AKWlpWpiExGJSKaazh6n5QT+pcDv2q5gZsPCGgtmNhI4EVgXzo8On/OAa4EfhPOHWnhFkpm9i6BX2u5IP4mIiLQrU9fRLAAeNrNPAW8AHwIIz7v8p7t/GpgK/NDM4gSBuMDd14Xvv8bMzg6X3+3uT4fLLwD+w8zqgX3ARe7tXCosIiKRM30PB01ny5cvz3QxRET6FTNb4e6lHa6noAEz20lwPU5XjCToCdfX9NVyQd8tm8rVOSpX5wzEck1291EdraSg6SYzW55Oove2vlou6LtlU7k6R+XqnAO5XBqIQ0REIqWgERGRSClouu+eTBcghb5aLui7ZVO5Okfl6pwDtlw6RyMiIpFSjUZERCKloEmTmf3UzHaY2doUr5uZfcfM1pvZmvDOBH2hXCeb2V4zWxU+buqFMk00s2fM7CUze9HMrkqyTq8frzTLlYnjlW9my8xsdViu+UnWyTOzh8Lj9byZTekj5fqEme1MOF6fjrpcCfvOMrN/mdkfkrzW68crzXJl8nhtNLMXwv3ud+FgpH+T7q5HGg/gJOBdwNoUr58F/IlgKO7jgOf7SLlOBv7Qy8dqHPCucLoIeAU4MtPHK81yZeJ4GVAYTucAzwPHtVnnCuAH4fTFwEN9pFyfAL7bm8crYd//DTyQ7N8rE8crzXJl8nhtBEa283pkf5Oq0aTJ3RcDb7WzyrnAzz2wFBga3jA00+Xqde6+1d1XhtMVwEvAhDar9frxSrNcvS48BpXhbE74aHvyNHEMp18Dpzbd1y/D5coIMysB3k/qwQ17/XilWa6+LLK/SQVNz5kAvJkwX0Yf+BILHR82f/zJzI7qzR2HTRazCH4NJ8ro8WqnXJCB4xU2t6wiuJP5k+6e8ni5ewOwl2AQwEyXC+CCsKnl12Y2McnrUbgD+BLBsO3JZOR4pVEuyMzxguBHwkIzW2HBMCltRfY3qaDpOcl+LfWFX38rCW4T8U7gLuCx3tqxmRUCvwGudvfyti8neUuvHK8OypWR4+XujR4MQV4CHGtm09qskpHjlUa5fg9McfcZwFO01CIiY8ENdXe4+4r2VkuyLNLjlWa5ev14JTjR3d8FnAl83oIxuxJFdswUND2njISRPgn+MLdkqCzN3L28qfnD3Z8gGJ10ZNT7NbMcgi/zX7r7b5OskpHj1VG5MnW8Eva/B3gWOKPNS83Hy8yygSH0YpNpqnK5+253rw1nfwQc3QvFORE4x8w2Ar8CTjGzX7RZJxPHq8NyZeh4Ne27aXyvHcCjwLFtVonsb1JB03MeJxiiwMzsOGCvh8NVZ5KZjW1qmzazYwn+zSMdoyfc30+Al9z9/1Ks1uvHK51yZeh4jTKzoeH0IIJRYV9us1riGE4XAk97eAY3k+Vq04Z/DsF5r0i5+/XuXuLuUwhO9D/t7h9rs1qvH690ypWJ4xXut8DMipqmgdOAtj1VI/ubzNR4NP2OmT1I0CNppJmVATcTnBzF3X8APEHQa2M9UA1c1kfKdSHwOTNrIBij5+Ko/+AIftl9HHghbN8H+B9gUkK5MnG80ilXJo7XOOA+M8siCLaH3f0PZnYLsNzdHycIyPvNbD3BL/OLIy5TuuW60szOIRhO/S2CXlUZ0QeOVzrlytTxGkMwxD0E3/sPuPufzew/Ifq/Sd0ZQEREIqWmMxERiZSCRkREIqWgERGRSCloREQkUgoaERGJlIJGREQipaARScHMzjGz63pwe8+aWWmS5aVm9p1w+hNm9t0U769MtrwHyvVE04WZ7ayTquwzzeysKMolA4cu2BRJIbzA7vFe2M9yYL/xQXqLu3cnKGYCpQQX+4kkpRqNHJDMbIqZvWxmPzaztWb2SzN7n5k9Z2avmtmxibULM7vXgkGh/mFmr5nZhR1s/0sWDDK12swWJLz0IQsGE3vFzOaE655syQfJOsjMlpjZP83sqx3s7/vhFeeY2aNm9tNw+lNmdms4/bFw36vM7IfhFf9NA2KNDKdvDI/Lk2b2oJnNS1V2M8sFbgEuCrd5UftHXQ5UCho5kB0K3AnMAI4APgK8G5hHcGuatsaFr58NLEjyOgBmdiZwHjA7vAv07QkvZ7v7scDVBLcLas+dwN3ufgywrYN1FwNzwukJwJHh9LuBv5nZVOAigjv4zgQagY+2KXcpcAHB8AkfJKipJGpVdnevA24iGFRsprs/1EEZ5QCloJED2evu/oK7x4EXgUXhfc1eAKYkWf8xd4+7+zqCe0el8j7gZ+5eDeDuiXcNbrpj9IoU+0h0IvBgOH1/B+v+DZhjZkcC64Dt4Q0cjwf+AZxKcKfgf4b3eTsVOLjNNt4N/M7d94UDw/2+zeudKbtIM52jkQNZbcJ0PGE+TvK/jcT12xut0Ug9jkfTNhpT7KOttG5G6O6bzWwYwW38FwPDgQ8Dle5eEd6R+j53v76Dcrens2UXAVSjEYnCQuCTZjYYwMyGd3E7z9Fy1+GPtrdiaAlBs9ZighrOvPAZYBFwoZmNbiqTmU1u8/6/Ax8ws3wLBod7fxr7rACK0lhPDmAKGpEe5u5/JuittjxspprXwVtSuYpgJMR/Egzc1ZG/EZxHWU8wUujwcBlhc9+XCYbyXQM8SXDOKbHc/wzLvZqgmWw5wRDI7XkGOFKdAaQ9GiZARJqZWaG7V4a1scXA5e6+MtPlkv5N7awikuiesENBPsE5HYWMdJtqNCJdZGbT2b83WK27zx5I+xTpLgWNiIhESp0BREQkUgoaERGJlIJGREQipaAREZFIKWhERCRS/x9jDQ/h1QmFVwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch2_2.best_score_, gsearch2_2.best_params_))\n",
    "test_means = gsearch2_2.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch2_2.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch2_2.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch2_2.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch2_2.cv_results_).to_csv('my_preds_maxdepth_min_child_weights_2.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(max_depth), len(min_child_weight))\n",
    "train_scores = np.array(train_means).reshape(len(max_depth), len(min_child_weight))\n",
    "\n",
    "for i, value in enumerate(max_depth):\n",
    "    pyplot.plot(min_child_weight, test_scores[i], label= 'test_max_depth:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'min_child_weight' )                                                                                                      \n",
    "pyplot.ylabel( '-Log Loss' )\n",
    "pyplot.savefig( 'max_depth_vs_min_child_weght_2.png' )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " ### 重新调整弱学习器数目"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def modelfit(alg, X_train, y_train, useTrainCV=True, cv_folds=None, early_stopping_rounds=100):\n",
    "    \n",
    "    if useTrainCV:\n",
    "        xgb_param = alg.get_xgb_params()\n",
    "        xgb_param['num_class'] = 9\n",
    "        \n",
    "        xgtrain = xgb.DMatrix(X_train, label = y_train)\n",
    "        \n",
    "        cvresult = xgb.cv(xgb_param, xgtrain, num_boost_round=alg.get_params()['n_estimators'], folds =cv_folds,\n",
    "                         metrics='mlogloss', early_stopping_rounds=early_stopping_rounds)\n",
    "        \n",
    "        n_estimators = cvresult.shape[0]\n",
    "        alg.set_params(n_estimators = n_estimators)\n",
    "        \n",
    "        print (cvresult)\n",
    "        #result = pd.DataFrame(cvresult)   #cv缺省返回结果为DataFrame\n",
    "        #result.to_csv('my_preds.csv', index_label = 'n_estimators')\n",
    "        cvresult.to_csv('my_preds4_2_3_232.csv', index_label = 'n_estimators')\n",
    "        \n",
    "        # plot\n",
    "        test_means = cvresult['test-mlogloss-mean']\n",
    "        test_stds = cvresult['test-mlogloss-std'] \n",
    "        \n",
    "        train_means = cvresult['train-mlogloss-mean']\n",
    "        train_stds = cvresult['train-mlogloss-std'] \n",
    "\n",
    "        x_axis = range(0, n_estimators)\n",
    "        pyplot.errorbar(x_axis, test_means, yerr=test_stds ,label='Test')\n",
    "        pyplot.errorbar(x_axis, train_means, yerr=train_stds ,label='Train')\n",
    "        pyplot.title(\"XGBoost n_estimators vs Log Loss\")\n",
    "        pyplot.xlabel( 'n_estimators' )\n",
    "        pyplot.ylabel( 'Log Loss' )\n",
    "        pyplot.savefig( 'n_estimators4_2_3_232.png' )\n",
    "    \n",
    "    #Fit the algorithm on the data\n",
    "    alg.fit(X_train, y_train, eval_metric='mlogloss')\n",
    "        \n",
    "    #Predict training set:\n",
    "    train_predprob = alg.predict_proba(X_train)\n",
    "    logloss = log_loss(y_train, train_predprob)\n",
    "\n",
    "        \n",
    "    #Print model report:\n",
    "    print (\"logloss of train :\" )\n",
    "    print (logloss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     train-mlogloss-mean  train-mlogloss-std  test-mlogloss-mean  \\\n",
      "0               1.952914            0.001514            1.954640   \n",
      "1               1.778569            0.001603            1.781626   \n",
      "2               1.640757            0.001437            1.645255   \n",
      "3               1.527644            0.001416            1.533472   \n",
      "4               1.432550            0.001417            1.439310   \n",
      "5               1.350933            0.001096            1.358850   \n",
      "6               1.279803            0.000991            1.288431   \n",
      "7               1.217941            0.000732            1.227486   \n",
      "8               1.163210            0.000797            1.173551   \n",
      "9               1.114315            0.000824            1.125346   \n",
      "10              1.070551            0.000460            1.082363   \n",
      "11              1.031195            0.000413            1.043844   \n",
      "12              0.995948            0.000569            1.009465   \n",
      "13              0.964187            0.000666            0.978394   \n",
      "14              0.935287            0.000653            0.950262   \n",
      "15              0.909083            0.000528            0.924833   \n",
      "16              0.885037            0.000594            0.901528   \n",
      "17              0.863132            0.000678            0.880193   \n",
      "18              0.843039            0.000806            0.860843   \n",
      "19              0.824716            0.000935            0.843253   \n",
      "20              0.807717            0.001022            0.826877   \n",
      "21              0.792043            0.001097            0.811947   \n",
      "22              0.777800            0.001088            0.798272   \n",
      "23              0.764414            0.001058            0.785599   \n",
      "24              0.752105            0.001047            0.773917   \n",
      "25              0.740589            0.001060            0.762894   \n",
      "26              0.730120            0.001144            0.752964   \n",
      "27              0.720158            0.000925            0.743577   \n",
      "28              0.710926            0.000929            0.734906   \n",
      "29              0.702343            0.000921            0.726802   \n",
      "..                   ...                 ...                 ...   \n",
      "202             0.487149            0.000769            0.588987   \n",
      "203             0.486778            0.000818            0.588979   \n",
      "204             0.486348            0.000799            0.589005   \n",
      "205             0.485973            0.000782            0.588996   \n",
      "206             0.485558            0.000755            0.589004   \n",
      "207             0.485169            0.000789            0.588947   \n",
      "208             0.484756            0.000818            0.588935   \n",
      "209             0.484348            0.000829            0.588935   \n",
      "210             0.483984            0.000840            0.588864   \n",
      "211             0.483542            0.000896            0.588883   \n",
      "212             0.483209            0.000843            0.588875   \n",
      "213             0.482776            0.000852            0.588795   \n",
      "214             0.482343            0.000900            0.588743   \n",
      "215             0.481965            0.000925            0.588801   \n",
      "216             0.481626            0.000908            0.588779   \n",
      "217             0.481157            0.000849            0.588701   \n",
      "218             0.480673            0.000778            0.588696   \n",
      "219             0.480295            0.000805            0.588733   \n",
      "220             0.479901            0.000804            0.588666   \n",
      "221             0.479552            0.000774            0.588686   \n",
      "222             0.479095            0.000736            0.588707   \n",
      "223             0.478701            0.000777            0.588688   \n",
      "224             0.478343            0.000734            0.588696   \n",
      "225             0.477949            0.000786            0.588647   \n",
      "226             0.477526            0.000718            0.588702   \n",
      "227             0.477120            0.000740            0.588682   \n",
      "228             0.476711            0.000776            0.588653   \n",
      "229             0.476299            0.000834            0.588681   \n",
      "230             0.475944            0.000825            0.588695   \n",
      "231             0.475569            0.000805            0.588746   \n",
      "\n",
      "     test-mlogloss-std  \n",
      "0             0.001758  \n",
      "1             0.002048  \n",
      "2             0.002350  \n",
      "3             0.002553  \n",
      "4             0.002450  \n",
      "5             0.002487  \n",
      "6             0.002499  \n",
      "7             0.002328  \n",
      "8             0.002442  \n",
      "9             0.002469  \n",
      "10            0.002617  \n",
      "11            0.002694  \n",
      "12            0.002629  \n",
      "13            0.002588  \n",
      "14            0.002600  \n",
      "15            0.002774  \n",
      "16            0.002801  \n",
      "17            0.002759  \n",
      "18            0.002885  \n",
      "19            0.002802  \n",
      "20            0.002936  \n",
      "21            0.002882  \n",
      "22            0.002804  \n",
      "23            0.002722  \n",
      "24            0.002649  \n",
      "25            0.002682  \n",
      "26            0.002666  \n",
      "27            0.002619  \n",
      "28            0.002803  \n",
      "29            0.002927  \n",
      "..                 ...  \n",
      "202           0.004663  \n",
      "203           0.004685  \n",
      "204           0.004638  \n",
      "205           0.004585  \n",
      "206           0.004590  \n",
      "207           0.004545  \n",
      "208           0.004597  \n",
      "209           0.004561  \n",
      "210           0.004616  \n",
      "211           0.004697  \n",
      "212           0.004717  \n",
      "213           0.004737  \n",
      "214           0.004693  \n",
      "215           0.004745  \n",
      "216           0.004711  \n",
      "217           0.004569  \n",
      "218           0.004576  \n",
      "219           0.004603  \n",
      "220           0.004613  \n",
      "221           0.004646  \n",
      "222           0.004704  \n",
      "223           0.004710  \n",
      "224           0.004706  \n",
      "225           0.004723  \n",
      "226           0.004772  \n",
      "227           0.004812  \n",
      "228           0.004770  \n",
      "229           0.004751  \n",
      "230           0.004787  \n",
      "231           0.004841  \n",
      "\n",
      "[232 rows x 4 columns]\n",
      "logloss of train :\n",
      "0.4912286281418515\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEXCAYAAABCjVgAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XmcHHWd//HXp3vuK5kkE3IfhHAFECEQENEIIocrIIsoAgq6su5P1mV1VfTnrv68r3V1RXQjC/FAWBQUZFkvDrMIwUwgEEg4cmdCjkkyyWTuoz+/P6pmpjPpniOZmpqZfj8fj3pMdVV11aere/rddX3L3B0RERGARNwFiIjIyKFQEBGRbgoFERHpplAQEZFuCgUREemmUBARkW4KBZE0ZvYZM7s97jpE4qJQGGXMrMzMNpnZe9OGlZvZFjO7Mm3YQjN7yMzqzGyfma0xsy+bWWU4/noz6zSzhrDbYGZ/F3Hti82sJsplDEametz9K+7+NxEtb5OZvTWKeUdhuN6v0bZexjqFwijj7g3AjcB3zawqHPwNoNrdfwlgZm8AHgf+DBzv7uOBi4AO4HVps3vK3cvcvQy4EviGmb1+eF6JDIaZ5cVdg+QId1c3CjtgKXA3sBjYA0xNG/cE8L1+nn898ESvYX8B3pv2+FLgRWAfQcickDbuhHDYvnCaS9PGXQKsAQ4A24B/AkqBZiAFNITdtCyv6/vAf4fPfxqYN4D1cTzwB2Av8DJw1eHUA3we+Fn4vDmAAzcAW4E64MPAGcDz4Wu/NW0584BHw/djN3AXMD4c99NwWc3hsj45gHW8CfhUuKxWIC98vC18LS8D52dYF2cBO4Bk2rB3As+H/WcC1UA9sBP4dpZ1uhioyTJuHPAToBbYDHwWSITjksC/hutgI3BTuB7zssxrE/DWLOM+BKwL39cHuz4zgAH/BuwC9ofr6KRs73fc/6+jqYu9AHWH+cZBJbA9/Me7IW14KdAJLO7n+deTFgrhF90+4Njw8bFAI3ABkA98MvznLAgfrwM+Ez4+L/wHPC587nbg3LQ6Twv7s37JpNWxNPwCODP8ErwLuKef55QSfGnfED7ntHC9LBhsPWQOhR8CRcDbgBbg18BkYHr4pfTmcPpjwvVVCFQBy4DvpM37oC+/vtZx2vSrgJlAMXBc+DqnpdWXMTCB9cAFaY9/AdwS9j8FXBf2lwFnZZlH1veLIBAeAMrDOl4BPhiO+zDBl/KMcH3/kcMIhfBztTt8PwuB7wHLwnEXAiuB8QQBcQLhD6Ns77e6gXXafTRKuXsdwS/MEuD+tFGVBLsFd3QNMLNvhMcVGs3ss2nTnhUObyDYSvgp8Go47t3Af7v7H9y9HfgWwRfTGwh+iZYBX3P3Nnd/FHgIuDp8bjtwoplVuHuduz8zyJd3v7v/xd07CELh1H6m/ytgk7vf6e4d4fLuI9glNhT1fNHdW9z99wRf4ne7+y533wb8L/B6AHdfF66vVnevBb4NvLmP+fa1jrv8u7tvdfdmgrAvDF9Lvrtvcvf1WeZ9N+H7YWblBL+e705bH8eY2SR3b3D35YNZGWaWDGv/tLsfcPdNBFsG14WTXAV8191rws/p1wYz/zTXAHe4+zPu3gp8GjjbzOaEr6GcYAvR3H2tu29Pe31H8n7nNIXCKGVm1xL8Qvsj8PW0UXUEuymmdg1w9096cFzhVwS/pLssd/fxHhxTmAIsAL4SjptGsFugax4pgl+p08NxW8NhXTaH4wD+muBLaLOZ/cnMzh7ky9uR1t9EEEB9mQ0sCgNun5ntI/hCmTJE9exM62/O8LgMwMwmm9k9ZrbNzOqBnwGT+phvX+u4y9a08euAmwm2ZnaFy5qWZd4/B64ws0LgCuAZd+9a1gcJtlJeMrMVZvZXfdSYySSCLcTNacPS3/9p6XX36h+M3uungWDX3PTwh8itBLsad5rZEjOrCCc90vc7pykURiEzm0ywP/VDwN8CV5nZmwDcvZFgP/wVg5mnu+8k+HX9jnDQawRftl3LNILdGNvCcTPNLP3zMysch7uvcPfLCHax/Bq4t2sxg6lpELYCfwoDrqsrc/e/G+Z6vhrO8xR3rwCuJdi10aX38vpaxxmf4+4/d/c3hs9zDv5BkD7dGoIv1IuB9xKERNe4V939aoL18XXgl2ZWOvCXyW6CX+Oz04Z1v/8Eu29mpI2bOYh5p+u9fkqBifR8zv7d3U8n+DFzLPCJcHi291sGQKEwOt0K/NrdHws3mT8J/Cj8VUj4+ANmdksYIJjZDGButhma2USCg5EvhoPuBd5uZuebWT7wcYKDnU8ShE4j8EkzyzezxQRhco+ZFZjZNWY2LtwlUk+w2wOCX9gTzWzcEK2HLg8Bx5rZdWE9+WZ2hpmdMMz1lBMcRN5nZtMJv6TS7ASOTnvc1zo+hJkdZ2bnhe9zC8FWSmemaUM/Bz4KvIngmELXfK41s6pwy2RfODjrfMysKL0j2BK9F/hyeDr0bOBjBFtGXa/rH8xsupmNJzg43p/8XsvJC+u/wcxODV/zV4Cn3X1T+P4uCtdbY7g+Ovt5v2Ug4j6ooW5wHXA5wS+o8b2GPwJ8Oe3xIuBhgn/6fcALwJeBieH46wn+WbrOvNlFsM95cto83klwwHA/8CfCA7fhuAXhsP3hNO8MhxcAvyXYjVUPrADemPa8Owh2Aewj+9lHX0p7vJh+Dk6H0x1HcMZSbTj/RwmORQyqHjIfaM5Lm76GtIP4BF+En01bJyvD9bmK4Eu+Jm3ay4At4bL+aQDreBMHH5g+heDYzwGCg/EPZVqHadPPIvgC/+9ew38Wvt8NBD8CLs/y/MXh6+/dHUNw7Opn4freCvwLPWcf5RFsye4hOPvoHwm2LCzLcjZlWMaXwnEfJjho3vV6Z4TDzyc446iBnjO9yvp7v9X131m4gkVEImFmFwM/dPfZ/U4ssdPuIxEZUmZWbGaXmFleuBvtcwQnOcgooC0FGRXM7FzgfzKN8+DsKRkhzKyEYFfY8QTHPf4b+Ad3r4+1MBkQhYKIiHTT7iMREek26hrZmjRpks+ZMyfuMkRERpWVK1fudveq/qaLLBTMbCZB+yhTCE6LW+Lu3+01jQHfJbj6sAm43vu5JH3OnDlUV1dHU7SIyBhlZpv7nyraLYUO4OPu/kzY9spKM/uDB1dadrkYmB92i4AfhH9FRCQGkR1TcPftXb/63f0AsJaD23SB4GKen3hgOTDezKYiIiKxGJYDzWGrhq8naB4h3XQObiyrhkODQ0REhknkoWBmZQQNrd2c4Txly/CUQ86RNbMbzazazKpra2ujKFNERIg4FMLGqu4D7nL3+zNMUsPBLSjOIGjX5yDuvsTdF7r7wqqqfg+ei4jIYYosFMIzi/4TWOvu384y2YPA+yxwFrDfe26UISIiwyzKs4/OIbgT02ozWxUO+wxBy424+w8JWvG8hOAWhE0Et1MUEZGYRBYK7v4EmY8ZpE/jwEeiqiHdgZZ2tuxtYl5VGUX5yeFYpIjIqJMzzVx86zvfYu5/zGfH+tVxlyIiMmLlTCj89ZlHU2KtNDfs639iEZEclTOhUFga3HGxrbEu5kpEREau3AmFskoA2pv2x1yJiMjIlTOhUFw2HoAOhYKISFY5Ewol4yYAkGo5EHMlIiIjV86EQmm4pUCLthRERLLJmVBI5OXT5IXQqi0FEZFsciYUABqthESbQkFEJJucCoXmRCl57Q1xlyEiMmLlVCi0JErJ79CWgohINjkVCm15pRR0NsZdhojIiJVTodCeV0ZxSqEgIpJNToVCZ34ZRammuMsQERmxcioUUgUVlLlCQUQkm5wKBQrLKbNm2to74q5ERGREyq1QKKoAoPGAms8WEckkp0Ih0RUK9XtjrkREZGSKLBTM7A4z22VmL2QZP87MfmNmz5nZi2YW+f2Z80qCeyo0a0tBRCSjKLcUlgIX9TH+I8Aad38dsBj4VzMriLAeCsIb7bTo7msiIhlFFgruvgzoaz+NA+VmZkBZOG2kR4ALSrtutKNQEBHJJM5jCrcCJwCvAauBf3D3VKYJzexGM6s2s+ra2trDXmDXjXZ09zURkcziDIULgVXANOBU4FYzq8g0obsvcfeF7r6wqqrqsBdYUh5sKXQ2KxRERDKJMxRuAO73wDpgI3B8lAssqQi2FJ59ZUuUixERGbXiDIUtwPkAZnYUcBywIcoFFpaMo8MTnHr4GxsiImNaXlQzNrO7Cc4qmmRmNcDngHwAd/8h8EVgqZmtBgz4lLvvjqqesCgOWBnWogPNIiKZRBYK7n51P+NfA94W1fKzaUhUkN+mYwoiIpnk1BXNAM155RS2a0tBRCSTnAuF1vxxFHfUx12GiMiIlHOh0F4wntKUbskpIpJJzoVCqnA8FX4Ad4+7FBGRESfnQoHiCZRZCw1NutmOiEhvORcKVjoBgPq6aM9+FREZjXIuFPLLglBorNsVcyUiIiNPzoVCYcUkAJr2a0tBRKS3nAuFkjAUWhv2xFyJiMjIk3uhMD5o+KijQVsKIiK95VwoVEyYDECqsS7mSkRERp6cC4X84qClVJr7uimciEhuyrlQ6GopNamWUkVEDpF7oQA0JCvIa1MoiIj0lpOh0JysoLBdjeKJiPQW2f0URrLW9g5KUgoFEZHecnJLoX3CcVTQGHcZIiIjTk6GgpdOYgL1NLe2x12KiMiIElkomNkdZrbLzF7oY5rFZrbKzF40sz9FVUtvibLJ5Fsne/eo/SMRkXRRbiksBS7KNtLMxgO3AZe6+wLgXRHWcpCCcUcBsH/3a8O1SBGRUSGyUHD3ZUBfV4i9F7jf3beE0w/bz/bi8UEoNO3dMVyLFBEZFeI8pnAsUGlmj5vZSjN7X7YJzexGM6s2s+ra2tojXnDpxGkAtOxXKIiIpIszFPKA04G3AxcC/2xmx2aa0N2XuPtCd19YVVV1xAseNykIhc76nUc8LxGRsSTO6xRqgN3u3gg0mtky4HXAK1EvuKiiik4MGtVSqohIuji3FB4AzjWzPDMrARYBa4dlyYkk+62CZLNCQUQkXWRbCmZ2N7AYmGRmNcDngHwAd/+hu681s98CzwMp4HZ3z3r66lBrSFZS0Kob7YiIpIssFNz96gFM803gm1HV0Jem/EpK2tR8tohIupy8ohmgtXAiFZ1qKVVEJF3OhkKqZBKVvp/2zlTcpYiIjBg5GwrVtUnKrZm9+/bHXYqIyIiRs6HwhoL1AOzdVRNzJSIiI0fOhkLJGz8MQH2tQkFEpEvOhkLF5FkAtOxVKIiIdMnZUBh/1GwAOvZtj7kSEZGRI2dDIVk6kTbySBxQ89kiIl1yNhQwoy4xkYJmNYonItIld0MBOFBQRUnrkTfFLSIyVuR0KLQUTWZ8pxrFExHpktOh0FE6hSqvo6mtI+5SRERGhJwOhWTFVMqshV27tbUgIgI5HgoFE2YAULdjS8yViIiMDDkdCmWTglBo3K1QEBGBHA+F8VOCC9geX/FczJWIiIwMOR0KpRNnArBoUkvMlYiIjAyRhYKZ3WFmu8ysz1tsmtkZZtZpZldGVUtWBSXstwryDmwb9kWLiIxEUW4pLAUu6msCM0sCXwd+F2EdfarLn0JZs5q6EBGBCEPB3ZcB/d0E+e+B+4BdUdXRn+aSaUzo2Im7x1WCiMiIEdsxBTObDrwT+GFcNQB0VsxgGrupa2yLswwRkREhzgPN3wE+5e6d/U1oZjeaWbWZVdfWDm1bRckJsyi2NrZv130VRETiDIWFwD1mtgm4ErjNzC7PNKG7L3H3he6+sKqqakiLKKmaC8C+7euHdL4iIqNRXlwLdve5Xf1mthR4yN1/Pdx1TJg2D4Dm2k3DvWgRkREnslAws7uBxcAkM6sBPgfkA7h7rMcR0pUdFWRT515d1Swi0m8omNk8oMbdW81sMXAK8BN339fX89z96oEW4e7XD3TaoWbFlTRRTPKAjimIiAzkmMJ9QKeZHQP8JzAX+HmkVQ0nM+ryj6KkSdcqiIgMJBRS7t5BcProd9z9H4Gp0ZY1vBpLpjOhfTuplK5VEJHcNpBQaDezq4H3Aw+Fw/KjK2n4dY6fw0x2sqtebSCJSG4bSCjcAJwNfNndN5rZXOBn0ZY1vPKr5lFqrWzbtjnuUkREYtXvgWZ3XwN8FMDMKoFyd/9a1IUNp4qpxwKwr+ZlWHB8zNWIiMSn3y0FM3vczCrMbALwHHCnmX07+tKGz4SZQRC07FoXcyUiIvEayO6jce5eD1wB3OnupwNvjbas4ZU3YTadJEjUbYy7FBGRWA0kFPLMbCpwFT0HmseWvAL25E2muEEXsIlIbhtIKHyB4H4H6919hZkdDbwabVnD70DxTCa21qgJbRHJaf2Ggrv/wt1Pcfe/Cx9vcPe/jr604fVC80RmsJPahta4SxERic1ADjTPMLNfhbfW3Glm95nZjOEobjidXLCDSmtg01Y1dyEiuWsgu4/uBB4EpgHTgd+Ew8aU8Rd8DIC9m/q8pbSIyJg2kFCocvc73b0j7JYCQ3tTgxGgctbJALTsWBtzJSIi8RlIKOw2s2vNLBl21wJ7oi5suFnlbNrIJ2+vrlUQkdw1kFD4AMHpqDuA7QR3SbshyqJikUiyu3AW4xt1rYKI5K6BnH20xd0vdfcqd5/s7pcTXMg25jSNm8fMzq3sb26PuxQRkVgc7j2aPzakVYwQiarjmGm1rH9td9yliIjE4nBDwYa0ihGiYsYCEubc9sv/ibsUEZFYHG4o9HvZr5ndEV7bkPEcTzO7xsyeD7snzex1h1nLkJkw5yQALjqqPuZKRETikbXpbDM7QOYvfwOKBzDvpcCtwE+yjN8IvNnd68zsYmAJsGgA841MoupYOklgu3Vaqojkpqyh4O7lRzJjd19mZnP6GP9k2sPlQPxXSecVsrtoNhMaXsXdMRuTe8lERLI63N1HQ+2DQNYd+WZ2o5lVm1l1bW1tpIU0VZ7AfN/Mtn3NkS5HRGQkij0UzOwtBKHwqWzTuPsSd1/o7gurqqK9mDp/2snMsN28unlbpMsRERmJYg0FMzsFuB24zN1HxFXSE+edBsCeDc/GXImIyPCLLRTMbBZwP3Cdu78SVx29Fc84BYBXVy+PuRIRkeGX9UBzlyxnIe0HqoGPu/uGLM+7G1gMTDKzGuBzQD6Au/8Q+BdgInBbeEC3w90XHt7LGELlU2lIVLDAdBc2Eck9/YYC8G3gNeDnBKejvgeYArwM3EHwxX8Id7+6r5m6+98AfzOIWoeHGfsrjmP23vXsbWxjQmlB3BWJiAybgew+usjd/8PdD7h7vbsvAS5x9/8CKiOuLx7TTuV428LqzbvirkREZFgNJBRSZnaVmSXC7qq0cWPyhsYT5i+i0DrY/qoONotIbhlIKFwDXAfsCrvrgGvNrBi4KcLaYlM8Ozi00bZ1ZcyViIgMr36PKYQHkt+RZfQTQ1vOCFE5h8ZEORV7V8ddiYjIsOp3S8HMZpjZr8LG7Xaa2X1mFn+TFFEyYy1HM69jHdv368pmEckdA9l9dCfwIDANmA78Jhw2ps06+RyOt608s35H3KWIiAybgYRClbvf6e4dYbcUiLatiRFgwrFvIN862fGSLmITkdwxkFDYbWbXmlky7K4FRkSTFFHKmx204m01T8dciYjI8BlIKHwAuArYAWwHrgRuiLKoEaFsMnVFM5je8AKNrR1xVyMiMiz6DQV33+Lul7p7lbtPdvfLgSuGobbYtU5ZyGn2Cs9urou7FBGRYXG4DeJ9bEirGKHGH/sGqmw/L72kU1NFJDccbijkxC3Jio5+AwBNz9wbcyUiIsPjcENhTDZvcYjJC2hOVjC1c7uOK4hITsgaCmZ2wMzqM3QHCK5ZGPsSCRqnnsmZtoZqHVcQkRyQNRTcvdzdKzJ05e4+kCa3x4SK489jdmIXL7z4QtyliIhELvZ7NI90Bce8CYCmV5fFXImISPQUCv2ZvICWvHHMqa9mZ31L3NWIiERKodCfRIK2WedybnI1f3pJN90RkbEtslAwszvCllUz7oy3wL+b2Toze97MTouqliNVftKFTLE6Xl79l7hLERGJVJRbCkuBi/oYfzEwP+xuBH4QYS1HxOadD0Dexkdp60jFXI2ISHQiCwV3Xwbs7WOSy4CfeGA5MN7MpkZVzxEZN52Gccdyrq3iqQ1jvi1AEclhcR5TmA5sTXtcEw47hJndaGbVZlZdW1s7LMX1VpRqYlHiJR57fkMsyxcRGQ5xhkKmpjIyXint7kvcfaG7L6yqiudWDnlX/oh866R5ze/oTOXGBd0iknviDIUaYGba4xnAazHV0r+Zi2gtqOTs9qd4x/f+N+5qREQiEWcoPAi8LzwL6Sxgv7tvj7GeviWSJI67iPMSqzhzdkXc1YiIRCLKU1LvBp4CjjOzGjP7oJl92Mw+HE7yMLABWAf8CPg/UdUyVPIXvIMKa6L2+T/Q3qmzkERk7ImsDSN3v7qf8Q58JKrlR2LeeXTklXBuyxP8ed37WXzc5LgrEhEZUrqieTDyi7Hj385FyRV86r+q465GRGTIKRQGKXnylYy3Rl7X9ix7G9viLkdEZEgpFAZr3nl0Wj5vtyf45cqt/U8vIjKKKBQGK6+A5MLruTi5gt8sf5GUrlkQkTFEoXA4Tn8/BbRz+v4/8OR6NXshImOHQuFwTDmZ1NTXc03+Y9y1fFPc1YiIDBmFwmFKnP5+5rOVXS89yS7dfEdExgiFwuE6+UpS+SW8K/EoS5/cFHc1IiJDQqFwuArLSZx0BZfnPcV9y1+iobUj7opERI6YQuFInP4BiryFS9r/yF3LN8ddjYjIEVMoHIkZp8Psc/hI0W+5/fGXqG9pj7siEZEjolA4Um/8GJM6a3lT6zJ+tEw34BGR0U2hcKSOOR+OOpmPFDzEbY+9wq4DOhNJREYvhcKRMoM33szR1HBB4hm+98i6uCsSETlsCoWhcOLlUDmXjxY8wM+Wb2J9bUPcFYmIHBaFwlBI5sGbPsGJvp4ri1bwmftXE9wuQkRkdFEoDJXXvQeOOpnPFf+CVRt38IvqmrgrEhEZtEhDwcwuMrOXzWydmd2SYfwsM3vMzJ41s+fN7JIo64lUIgkXfomy5m18dtIyvvzwWnY3tMZdlYjIoER5j+Yk8H3gYuBE4GozO7HXZJ8F7nX31wPvAW6Lqp5hcfRimH8h7237BUVte/n8gy9qN5KIjCpRbimcCaxz9w3u3gbcA1zWaxoHKsL+ccBrEdYzPN72RZJtB7ij+Ls89Px27UYSkVElylCYDqTfmqwmHJbu88C1ZlYDPAz8fYT1DI+q4+Ccm1nQsYYLi17klvuf5+UdB+KuSkRkQKIMBcswrPe+lKuBpe4+A7gE+KmZHVKTmd1oZtVmVl1bWxtBqUNs8adh0rHcVv5jZpSk+D93raRRDeaJyCgQZSjUADPTHs/g0N1DHwTuBXD3p4AiYFLvGbn7Endf6O4Lq6qqIip3COUXwWXfJ3lgG/ce/TDraxt549cf1a07RWTEizIUVgDzzWyumRUQHEh+sNc0W4DzAczsBIJQGAWbAgMw80won8qUV3/OkkW7qWtq5ysPr427KhGRPkUWCu7eAdwE/A5YS3CW0Ytm9gUzuzSc7OPAh8zsOeBu4HofS6frfPRZmHIKF7z8z9x8egG3P7GRH+uGPCIygtlo+w5euHChV1dXx13GwO3dCN87Hc8rZlHHf7Cr2fjWu17HlafPiLsyEckhZrbS3Rf2N52uaI7ahLnw7p9h7Q08ecrDnHvMRD7xy+e45y9b4q5MROQQCoXhcPwl8KZPkPfcXdxx9OO8+dgqbrl/NUv/vDHuykREDqJQGC5v+b9QOpn8ZV/lRye9yAUnHsXnf7OGL/xmDZ06K0lERgiFwnAxg4+tgWPeSv7DH+OHp9cwpaKQO/68kRt/Us0B3cpTREYAhcJwSubDu34MM84g+csPsPwd+/jiZQt45KVdLPzSH1m1dV/cFYpIjlMoDLfCMrj2Pph1Ftz/Ia4rfpJffPhs3OGd3/8z339sHR2dqbirFJEcpVCIQ2E5XPNLKKyAX/8dZ2y5gxWfOZ/K0gK++buXuez7f2Z1zf64qxSRHKRQiEtBCXz8ZTj5Knj0i4z73d+z8pZz+cE1p/HKzgO849Yn+Odfv6B7MojIsFIoxCm/CK5YEpyZ9Pw92NJLuHhGK9WfvYDrzprNT5dv5swv/5Hv/PEVGtSgnogMA13RPFKseQAe+HvA4R3fgZP+mg21DXzr9y/z8Ood5CWMT150HO8+YxbjivPjrlZERpmBXtGsUBhJ6jbDfR+EmhWw4Aq46KtQPoVVW/fxvv98mvqWDhIG1501m+vPmcvcSaVxVywio4RCYbTqbIcnvgOPfRksAZd8E06/ARIJXti2n+vv/At7GtqCW9YV5fHFy0/iwgVTKMpPxl25iIxgCoXRbs96eOhm2LgMZpwJF30NZpwOwK4DLfz86S3c9vh62jpSJBPGu8+YyeWnTmfh7EoSiUz3NxKRXKZQGAvc4fn/gt9/Fhpr4cTL4Lx/hknzAUilnOUb9vCLlTU8sGobKYf8pHH1mbNYfFwVi+ZOpLQwL+YXISIjgUJhLGk9AE/eCsu+AZ4KwuGcm2H6ad2TNLR28OhLu/jcAy+wv7mdlAf3Qz3r6Im8cf4kFs2dwMkzxlGYp91MIrlIoTAWNeyC5bfBijugdT/MfRO84aMw73xI9Jxd3NLeycrNdSx7pZafLt9MU1snEDS/tHB2JQvnTOCMOZWcNH0ck8uL4no1IjKMFApjWUs9rFwKj34ROtugci6c8UE49RoomXDI5HsaWlm5uY7qzXXc9fRmmlo76XrX85PG2fMmccLUck6cWsH8yeXMnlii3U4iY4xCIRd0tMHaB2HF7bDlqeBspZPfFZzOOu88yCvI+LTmtk6eq9nHmtfqWbO9njWv1bN2ez3pn4T8pFGYl+TCBVOYPbGE2RNLmDOxlOmVxUwoKdDBbJFRZkSEgpldBHwXSAK3u/vXMkxzFfB5wIHn3P29fc1ToZDFjheCcHj2J5DqhKJxcPxfwYJ3wtGLgxZa+9DWkWJ9bQMbahvZtKeRzXsa2byniWe21NHeeehnpDAvwSkzxjFlXDFS7f2kAAAO+klEQVRTxxUxpaKIKeOCrqqskAmlBZQUJDFTeIiMBLGHgpklgVeAC4AaYAVwtbuvSZtmPnAvcJ6715nZZHff1dd8FQr96GiDDY/Di/fD8/eChwEx73w49kI45q1QOmlQs2xu62TL3iY27Wlk+75mtte3sHN/C4+8tIuG1g6yfYS64qAoP8mpM8dTWZpPZUkBlSUFjC/Jp7Qwj9LCPMoKk5QU5FEWPi4tTFJWmEdxvkJFZKiMhFA4G/i8u18YPv40gLt/NW2abwCvuPvtA52vQmEQOlph/aOw9jfw6h+gMczb6Qth/tuCLYhpr8+6m2mg3J26pnZ27G9h+/5mdje0UtfUTl1TG3WNbdQ1tbOvqY3V2/bT2p5iMJ+4pBkpdwrzExx3VHl3kJQWJMNACR6XFCQpyEtQkExQmJ+gIBk8LsxLHPK3MK/3tAnykmoGTMa2kRAKVwIXufvfhI+vAxa5+01p0/yaYGviHIJdTJ93999mmNeNwI0As2bNOn3z5s2R1DympVKwfVUQDq/+HraFwWoJmPtmmHNOEBbTToXiyohLcRraOmhsDbqG1k6aWjtoaO2gsS143DMu+PvHtbvoTDmplNPY1oETBEane9YtlcORMDAzUuEtUvOShmG0p4J7XBTmJTCM1o5OivOTHDulnKQZyURPl3dQf4JEOCxh4bikdT8nfdr05x/ynHBYwqy7xq7+hBkW/j14fDgsMcjpw/FAz99wu88syzDSp7dez6V7iy99Ous13UHPMes1z2D69HnSZx2Zn88hr+vg2jK9tkNq61VHxtc2ArdwBxoKUZ5ikmmt9P73zQPmA4uBGcD/mtlJ7n7QLcjcfQmwBIIthaEvNQckEsF1DdNPg8WfgsY9sOVJ2PRE0D36pZ5pJxwdbEF0dVNOgaKKISzFqCjKp6JoaBr2a+9M0dLeSVtHitaOFG0dKdo6U7S2p2jr7KS1PUVrZ+qg8a0dwfS9n9OZ8u6uIwyhjpST8uBvZyrFU+v30NaZoqUjxYbaBtyhMWzFtqggCQ7N7cFpwAV5CXBo7Ux1f/rzkoY7dITBYwb4of8cMnaFb3l3P+mPwwHpP3a6zuuYOq6IP99yfqS1RRkKNcDMtMczgNcyTLPc3duBjWb2MkFIrIiwLgEonQgnvCPoAJrr4LVV8Nqz8NozsPUv8MJ9PdNXzIDJx8PkE6DqhKB/0nHBneRilp9MkD9Gdv+kUk6nHxxMwd8gVFIOKQ9Cyrv7CR/39KdS9JomGHfQNNnm0fXcsKaeL6eerbL0cV1TuqcP7/lG63lO2nRp8+maNm0xB03bM13PfHrXQa96e89zILX199ro9fyD5zmI2vp5bT3roGd5XePPnjeRqEUZCiuA+WY2F9gGvAfofWbRr4GrgaVmNgk4FtgQYU2STXElzHtL0HVpqA12OW1/Dmpfgl0vwbpHOOg37fhZPSEx4eigq5wLFdMPuqBOBiaRMBIYat9Q4hJZKLh7h5ndBPyO4HjBHe7+opl9Aah29wfDcW8zszVAJ/AJd98TVU0ySGVVMP+CoOvS2QF1m6B2LewKu9qXggPaqfae6ZKFUDm7JyQq58C46UFYjJsBJZMUGiIjkC5ek6GR6oT9NVC3EfZugL0bw/6wa288ePpkAZRPDQKiYjpUTOvpLz8KSquCLr84ntcjMsaMhAPNkksSyWDLoHJ2cKprOndo3A3124Ju/zaor4H614L+rcuhfvvBWxpdCsqD6ypKq6Bsck9/6eSDh5dMDHaBJbTfReRIKBQkembBrqiyquCU10xSqaB58Pqa4FhG467gcUNt8LexNtgC2fo0NO0JWovNpGgcFE8I2oBK/1tcCcXjg79F44P+9L9HeK2GyFihUJCRIZEIdhuVH9X/tKlOaNrbExaNtcGWSPPe4Cyqpr1Bf2Mt7H4Zmuqg7UDf88wv6QmJbMHR/bfy4GH9NCEiMpooFGT0SSR7tjwGqrMdWvZD8z5o2Rf8ba7r6e/9d9/m4Kyr5rpDj4f0ll+aOTiKKqCwvKcrKIPCXsO6OgWLjBAKBckNyfzwGMTg2n0CgvakWvZnD5CWMGC6+us2BafyttRDWwMDuiwtr6hXgGQIjsIsoVJQDgWlYVcGSf1by+HTp0ekP3kFg98y6ZJKBVsarQfSuvrwb0OGYQeCIGk9EBxf6RrWUp/5QHwmycIgIArLgpBID4zu/t6Pe/f3eqxjLjlDoSASpUSi5xf9kepoDYOkvlfIHAiCp62razi0v7UhOGifPq6jeRCvI39gwVLYV9D06s8rOrhxIRkRFAoio0VeYdCVDlFTB6nOtPDIEiaH9Hc9DofV1xw6bqAskRYWvf+WQkFJcLwmvzg4EaCgpKe/q8s0rOuxLo48LAoFkVyVSAYHw4ewsUNSqWALZEDBkmFrpq0RGnb2jGtvgvZm6GwdfC15RWFAdAVLcRA2B4VIH4GTXxwO7z0snMcY3dJRKIjI0Ekken7pM3no5tvZEYRNexg47WF/e1d/E7Q1hSHS1GtYc9rwpmB324GdB0/b1hjckGpQrJ9gSR+eIVT6Gtb1nBjOSlMoiMjIl8yD5BAdm8mmo+3gQDkoaJoPDZGsw5qhaXfmYBpsA+mJvIPDY+EH4JyPRvLyuygUREQgOMMqryC41iQK7tDR0muLJj00em/1NPcKpOagjbCIKRRERIaDWc+xDaK/L8Lh0uF5ERHpplAQEZFuCgUREemmUBARkW4KBRER6RZpKJjZRWb2spmtM7Nb+pjuSjNzM+v3VnEiIhKdyELBzJLA94GLgROBq83sxAzTlQMfBZ6OqhYRERmYKLcUzgTWufsGd28D7gEuyzDdF4FvAC0R1iIiIgMQ5cVr04GtaY9rgEXpE5jZ64GZ7v6Qmf1TthmZ2Y3AjeHDBjN7+TBrmgTsPsznjiVaDwGth4DWQ2Csr4fZA5koylDI1Hxgd8MfZpYA/g24vr8ZufsSYMkRF2RW7e45f9xC6yGg9RDQeghoPQSi3H1UA8xMezwDeC3tcTlwEvC4mW0CzgIe1MFmEZH4RBkKK4D5ZjbXzAqA9wAPdo109/3uPsnd57j7HGA5cKm7V0dYk4iI9CGyUHD3DuAm4HfAWuBed3/RzL5gZpdGtdx+HPEuqDFC6yGg9RDQeghoPQDmPsj2vUVEZMzSFc0iItJNoSAiIt1yJhQG2uTGWGRmm8xstZmtMrPqcNgEM/uDmb0a/q2Mu86hZmZ3mNkuM3shbVjG122Bfw8/H8+b2WnxVT60sqyHz5vZtvAzscrMLkkb9+lwPbxsZhfGU/XQMrOZZvaYma01sxfN7B/C4Tn3eehPToTCQJvcGOPe4u6npp2HfQvwiLvPBx4JH481S4GLeg3L9rovBuaH3Y3AD4apxuGwlEPXA8C/hZ+JU939YYDw/+I9wILwObeF/z+jXQfwcXc/geD094+ErzUXPw99yolQYOBNbuSSy4Afh/0/Bi6PsZZIuPsyYG+vwdle92XATzywHBhvZlOHp9JoZVkP2VwG3OPure6+EVhH8P8zqrn7dnd/Juw/QHBG5HRy8PPQn1wJhUxNbkyPqZY4OPB7M1sZNhkCcJS7b4fgHwaYHFt1wyvb687Fz8hN4a6RO9J2H4759WBmc4DXEzTCqc9DL7kSCn02uZEDznH30wg2iT9iZm+Ku6ARKNc+Iz8A5gGnAtuBfw2Hj+n1YGZlwH3Aze5e39ekGYaNmfXQl1wJhf6a3BjT3P218O8u4FcEuwN2dm0Oh393xVfhsMr2unPqM+LuO929091TwI/o2UU0ZteDmeUTBMJd7n5/OFifh15yJRT6bHJjLDOz0vCeFZhZKfA24AWC1//+cLL3Aw/EU+Gwy/a6HwTeF551chawv2u3wljUa//4Owk+ExCsh/eYWaGZzSU40PqX4a5vqJmZAf8JrHX3b6eN0uehlyhbSR0x3L3DzLqa3EgCd7j7izGXNVyOAn4V/E+QB/zc3X9rZiuAe83sg8AW4F0x1hgJM7sbWAxMMrMa4HPA18j8uh8GLiE4sNoE3DDsBUcky3pYbGanEuwS2QT8LUDYFM29wBqCM3Y+4u6dcdQ9xM4BrgNWm9mqcNhnyMHPQ3/UzIWIiHTLld1HIiIyAAoFERHpplAQEZFuCgUREemmUBARkW4KBRER6aZQEBkAMzu1V/PSlw5VE+xmdrOZlQzFvESOlK5TEBkAM7seWOjuN0Uw703hvHcP4jnJMXJRmYww2lKQMcXM5oQ3UvlReDOV35tZcZZp55nZb8PWY//XzI4Ph7/LzF4ws+fMbFnYNMoXgHeHN6R5t5ldb2a3htMvNbMfhDdx2WBmbw5bHl1rZkvTlvcDM6sO6/p/4bCPAtOAx8zssXDY1RbcFOkFM/t62vMbzOwLZvY0cLaZfc3M1oQtnX4rmjUqOcfd1akbMx0wh6B5hlPDx/cC12aZ9hFgfti/CHg07F8NTA/7x4d/rwduTXtu92OCm9jcQ9Cy5mVAPXAywY+ulWm1TAj/JoHHgVPCx5uASWH/NILmFqoImiV5FLg8HOfAVV3zAl6mZ2t/fNzrXt3Y6LSlIGPRRnfvat9mJUFQHCRsQvkNwC/CtnD+A+hqJO7PwFIz+xDBF/hA/MbdnSBQdrr7ag9aIH0xbflXmdkzwLMEdzbLdPe/M4DH3b3W3TuAu4Cups47CVr5hCB4WoDbzewKgvZ5RI5YTjSIJzmnNa2/E8i0+ygB7HP3U3uPcPcPm9ki4O3AqrDhuIEuM9Vr+SkgL2xx9J+AM9y9LtytVJRhPpna8e/S4uFxBA8aeTwTOJ+g1d+bgPMGUKdIn7SlIDnJgxusbDSzd0H3jdpfF/bPc/en3f1fgN0E7eofAMqPYJEVQCOw38yOIrjhUZf0eT8NvNnMJoX3Rr4a+FPvmYVbOuM8uLfyzQQ3yxE5YtpSkFx2DfADM/sskE9wXOA54JtmNp/gV/sj4bAtwC3hrqavDnZB7v6cmT1LsDtpA8Euqi5LgP8xs+3u/hYz+zTwWLj8h909070uyoEHzKwonO4fB1uTSCY6JVVERLpp95GIiHTT7iMZ88zs+wR33kr3XXe/M456REYy7T4SEZFu2n0kIiLdFAoiItJNoSAiIt0UCiIi0u3/A5mSYwML2NADAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#调整max_depth和min_child_weight之后再次调整n_estimators(6,4)\n",
    "xgb2_3 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=232,  #数值大没关系，cv会自动返回合适的n_estimators\n",
    "        max_depth=6,\n",
    "        min_child_weight=4,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel=0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "modelfit(xgb2_3, X_train, y_train, cv_folds = kfold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-0.58409673 -0.58733472 -0.58951265 -0.58924584 -0.59195808]\n",
      "CV logloss: -58.84% (0.26%)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "results = cross_val_score(xgb2_3, X_train, y_train, scoring='neg_log_loss', cv=kfold)\n",
    "print (results)\n",
    "print (\"CV logloss: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 调整 subsample & colsample_bytree 参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'subsample': [0.6, 0.7, 0.8, 0.9, 1.0],\n",
       " 'colsample_bytree': [0.4, 0.5, 0.6, 0.7, 0.8]}"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#max_depth 建议3-10， min_child_weight=1／sqrt(ratio_rare_event) =5.5\n",
    "subsample = [i/10.0 for i in range(6,11)]\n",
    "colsample_bytree = [i/10.0 for i in range(4,9)]\n",
    "param_test3_1 = dict(subsample=subsample, colsample_bytree=colsample_bytree)\n",
    "param_test3_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.58488, std: 0.00376, params: {'colsample_bytree': 0.4, 'subsample': 0.6},\n",
       "  mean: -0.58353, std: 0.00345, params: {'colsample_bytree': 0.4, 'subsample': 0.7},\n",
       "  mean: -0.58262, std: 0.00308, params: {'colsample_bytree': 0.4, 'subsample': 0.8},\n",
       "  mean: -0.58200, std: 0.00322, params: {'colsample_bytree': 0.4, 'subsample': 0.9},\n",
       "  mean: -0.58168, std: 0.00376, params: {'colsample_bytree': 0.4, 'subsample': 1.0},\n",
       "  mean: -0.58299, std: 0.00332, params: {'colsample_bytree': 0.5, 'subsample': 0.6},\n",
       "  mean: -0.58253, std: 0.00348, params: {'colsample_bytree': 0.5, 'subsample': 0.7},\n",
       "  mean: -0.58246, std: 0.00380, params: {'colsample_bytree': 0.5, 'subsample': 0.8},\n",
       "  mean: -0.58178, std: 0.00357, params: {'colsample_bytree': 0.5, 'subsample': 0.9},\n",
       "  mean: -0.58172, std: 0.00392, params: {'colsample_bytree': 0.5, 'subsample': 1.0},\n",
       "  mean: -0.58359, std: 0.00387, params: {'colsample_bytree': 0.6, 'subsample': 0.6},\n",
       "  mean: -0.58192, std: 0.00392, params: {'colsample_bytree': 0.6, 'subsample': 0.7},\n",
       "  mean: -0.58107, std: 0.00363, params: {'colsample_bytree': 0.6, 'subsample': 0.8},\n",
       "  mean: -0.58102, std: 0.00347, params: {'colsample_bytree': 0.6, 'subsample': 0.9},\n",
       "  mean: -0.58194, std: 0.00435, params: {'colsample_bytree': 0.6, 'subsample': 1.0},\n",
       "  mean: -0.58245, std: 0.00333, params: {'colsample_bytree': 0.7, 'subsample': 0.6},\n",
       "  mean: -0.58169, std: 0.00303, params: {'colsample_bytree': 0.7, 'subsample': 0.7},\n",
       "  mean: -0.58131, std: 0.00417, params: {'colsample_bytree': 0.7, 'subsample': 0.8},\n",
       "  mean: -0.58127, std: 0.00420, params: {'colsample_bytree': 0.7, 'subsample': 0.9},\n",
       "  mean: -0.58173, std: 0.00328, params: {'colsample_bytree': 0.7, 'subsample': 1.0},\n",
       "  mean: -0.58243, std: 0.00334, params: {'colsample_bytree': 0.8, 'subsample': 0.6},\n",
       "  mean: -0.58161, std: 0.00336, params: {'colsample_bytree': 0.8, 'subsample': 0.7},\n",
       "  mean: -0.58167, std: 0.00315, params: {'colsample_bytree': 0.8, 'subsample': 0.8},\n",
       "  mean: -0.58128, std: 0.00329, params: {'colsample_bytree': 0.8, 'subsample': 0.9},\n",
       "  mean: -0.58190, std: 0.00418, params: {'colsample_bytree': 0.8, 'subsample': 1.0}],\n",
       " {'colsample_bytree': 0.6, 'subsample': 0.9},\n",
       " -0.5810201280090288)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb3_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=231,  #第二轮参数调整得到的n_estimators最优值\n",
    "        max_depth=6,\n",
    "        min_child_weight=4,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch3_1 = GridSearchCV(xgb3_1, param_grid = param_test3_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch3_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch3_1.grid_scores_, gsearch3_1.best_params_,     gsearch3_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([173.64168701, 175.09037805, 174.6644166 , 175.7569972 ,\n",
       "        170.59234157, 213.66575694, 212.77159877, 199.93222961,\n",
       "        201.25076776, 203.96193447, 235.89934535, 230.59605489,\n",
       "        234.57125988, 233.56969876, 233.17192183, 272.81239581,\n",
       "        270.01489506, 287.33779697, 286.03915415, 277.06418371,\n",
       "        312.45475073, 308.55322862, 305.42846069, 303.98753986,\n",
       "        196.37085686]),\n",
       " 'std_fit_time': array([ 1.88658896,  1.59471899,  1.67882761,  0.81162174,  2.81075301,\n",
       "         2.24896106,  3.48069061,  5.94041532,  8.04310004, 16.43184373,\n",
       "        11.68960295,  5.2497927 ,  3.53689869,  4.30693356,  2.95403674,\n",
       "         2.39883564,  4.77815216,  2.82534003,  5.10318896,  6.40124442,\n",
       "         0.76570481,  4.78770854,  5.64587258,  4.20053522, 22.40020763]),\n",
       " 'mean_score_time': array([0.93990202, 1.16852155, 1.02918787, 0.9248878 , 1.02228112,\n",
       "        0.91057401, 0.91858172, 0.86212759, 0.85522075, 0.91217537,\n",
       "        0.91367683, 1.05491242, 0.9990591 , 0.85812359, 0.83750391,\n",
       "        0.8792439 , 0.89225645, 0.91047406, 0.90306721, 0.83820457,\n",
       "        0.91507845, 0.87954426, 0.91067429, 0.80447216, 0.60959244]),\n",
       " 'std_score_time': array([0.06911784, 0.17209009, 0.17806721, 0.02616895, 0.17971332,\n",
       "        0.05280159, 0.06133805, 0.01899606, 0.04868753, 0.13721825,\n",
       "        0.02619139, 0.16609872, 0.20603972, 0.01918531, 0.03015482,\n",
       "        0.0639092 , 0.10404761, 0.07796366, 0.05767249, 0.01701764,\n",
       "        0.10606125, 0.03243416, 0.05271806, 0.07324514, 0.02086223]),\n",
       " 'param_colsample_bytree': masked_array(data=[0.4, 0.4, 0.4, 0.4, 0.4, 0.5, 0.5, 0.5, 0.5, 0.5, 0.6,\n",
       "                    0.6, 0.6, 0.6, 0.6, 0.7, 0.7, 0.7, 0.7, 0.7, 0.8, 0.8,\n",
       "                    0.8, 0.8, 0.8],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_subsample': masked_array(data=[0.6, 0.7, 0.8, 0.9, 1.0, 0.6, 0.7, 0.8, 0.9, 1.0, 0.6,\n",
       "                    0.7, 0.8, 0.9, 1.0, 0.6, 0.7, 0.8, 0.9, 1.0, 0.6, 0.7,\n",
       "                    0.8, 0.9, 1.0],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'colsample_bytree': 0.4, 'subsample': 0.6},\n",
       "  {'colsample_bytree': 0.4, 'subsample': 0.7},\n",
       "  {'colsample_bytree': 0.4, 'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.4, 'subsample': 0.9},\n",
       "  {'colsample_bytree': 0.4, 'subsample': 1.0},\n",
       "  {'colsample_bytree': 0.5, 'subsample': 0.6},\n",
       "  {'colsample_bytree': 0.5, 'subsample': 0.7},\n",
       "  {'colsample_bytree': 0.5, 'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.5, 'subsample': 0.9},\n",
       "  {'colsample_bytree': 0.5, 'subsample': 1.0},\n",
       "  {'colsample_bytree': 0.6, 'subsample': 0.6},\n",
       "  {'colsample_bytree': 0.6, 'subsample': 0.7},\n",
       "  {'colsample_bytree': 0.6, 'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.6, 'subsample': 0.9},\n",
       "  {'colsample_bytree': 0.6, 'subsample': 1.0},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.6},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.7},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.9},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 1.0},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.6},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.7},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.9},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 1.0}],\n",
       " 'split0_test_score': array([-0.57930517, -0.57772332, -0.57744458, -0.57687662, -0.57562994,\n",
       "        -0.57685683, -0.57684012, -0.57644113, -0.57671681, -0.57580552,\n",
       "        -0.57682817, -0.5750859 , -0.57486804, -0.57545021, -0.57538612,\n",
       "        -0.57640997, -0.57698197, -0.5740382 , -0.57468981, -0.57631115,\n",
       "        -0.57736669, -0.57679947, -0.57704369, -0.57609353, -0.57490557]),\n",
       " 'split1_test_score': array([-0.58251218, -0.58203171, -0.58165488, -0.58153635, -0.58036923,\n",
       "        -0.58318514, -0.58169157, -0.58051109, -0.5794541 , -0.58000747,\n",
       "        -0.58228433, -0.58074683, -0.57940956, -0.57931819, -0.57990747,\n",
       "        -0.58165793, -0.5803979 , -0.57998788, -0.57941706, -0.58120385,\n",
       "        -0.58140145, -0.57916524, -0.57997556, -0.5797576 , -0.58037715]),\n",
       " 'split2_test_score': array([-0.58505283, -0.58414314, -0.58254699, -0.58080164, -0.5812313 ,\n",
       "        -0.58353407, -0.58170973, -0.58254287, -0.58120157, -0.58068455,\n",
       "        -0.58452175, -0.58253935, -0.58253039, -0.58104259, -0.58092501,\n",
       "        -0.58340352, -0.58188374, -0.58210364, -0.58090682, -0.5811097 ,\n",
       "        -0.58135747, -0.58199099, -0.58094677, -0.58095252, -0.58189208]),\n",
       " 'split3_test_score': array([-0.58743766, -0.58626992, -0.58609584, -0.58475925, -0.58477311,\n",
       "        -0.58455463, -0.58610525, -0.58574841, -0.58524158, -0.58569351,\n",
       "        -0.5863666 , -0.58490395, -0.58331405, -0.58443647, -0.58630556,\n",
       "        -0.5853346 , -0.58302576, -0.58445873, -0.58448287, -0.58395033,\n",
       "        -0.58491756, -0.58365736, -0.58525332, -0.58488711, -0.58613977]),\n",
       " 'split4_test_score': array([-0.59010517, -0.58749694, -0.58536406, -0.58602348, -0.58641626,\n",
       "        -0.58682709, -0.58628169, -0.58707441, -0.58629254, -0.58643351,\n",
       "        -0.58793142, -0.58632366, -0.58524887, -0.58485435, -0.58718937,\n",
       "        -0.58545982, -0.58617526, -0.58596343, -0.58683773, -0.58605708,\n",
       "        -0.58709726, -0.58641387, -0.58511305, -0.58473328, -0.5861703 ]),\n",
       " 'mean_test_score': array([-0.58488229, -0.58353276, -0.5826211 , -0.58199922, -0.58168368,\n",
       "        -0.58299132, -0.58252544, -0.5824633 , -0.58178105, -0.58172463,\n",
       "        -0.58358619, -0.58191967, -0.58107393, -0.58102013, -0.58194239,\n",
       "        -0.58245299, -0.58169265, -0.58131009, -0.58126652, -0.58172616,\n",
       "        -0.5824278 , -0.5816051 , -0.58166627, -0.5812846 , -0.58189671]),\n",
       " 'std_test_score': array([0.00375646, 0.00345198, 0.00307681, 0.00321559, 0.00375552,\n",
       "        0.00332024, 0.00348159, 0.00379832, 0.00356997, 0.00392402,\n",
       "        0.00386915, 0.00391816, 0.00362909, 0.00347255, 0.00435261,\n",
       "        0.00332864, 0.00302559, 0.00416712, 0.00419958, 0.00327648,\n",
       "        0.00334106, 0.00336239, 0.00314614, 0.00329425, 0.00418215]),\n",
       " 'rank_test_score': array([25, 23, 21, 16,  8, 22, 20, 19, 12, 10, 24, 14,  2,  1, 15, 18,  9,\n",
       "         5,  3, 11, 17,  6,  7,  4, 13]),\n",
       " 'split0_train_score': array([-0.48406616, -0.48221647, -0.48289977, -0.48124323, -0.4846692 ,\n",
       "        -0.47945011, -0.47837478, -0.47642268, -0.47694516, -0.4809335 ,\n",
       "        -0.47333114, -0.47270121, -0.47203705, -0.47549363, -0.48142527,\n",
       "        -0.47087173, -0.46945599, -0.46901294, -0.47290483, -0.47771787,\n",
       "        -0.46688595, -0.46663711, -0.46895091, -0.47170719, -0.47772842]),\n",
       " 'split1_train_score': array([-0.48293186, -0.4810745 , -0.48019559, -0.48124462, -0.48336426,\n",
       "        -0.47742838, -0.47578878, -0.47496472, -0.47478067, -0.47898361,\n",
       "        -0.47387449, -0.47085565, -0.47188262, -0.47255685, -0.47966008,\n",
       "        -0.46794408, -0.46703625, -0.46789067, -0.47101355, -0.47606991,\n",
       "        -0.46905408, -0.46677514, -0.46670965, -0.46769864, -0.47244434]),\n",
       " 'split2_train_score': array([-0.483733  , -0.48041247, -0.48019036, -0.48007498, -0.4834727 ,\n",
       "        -0.4764214 , -0.47564838, -0.47720295, -0.47650297, -0.47888413,\n",
       "        -0.47210377, -0.46974619, -0.47078038, -0.47332423, -0.4766361 ,\n",
       "        -0.46881765, -0.46791289, -0.46977096, -0.47130137, -0.47562398,\n",
       "        -0.46482158, -0.46331465, -0.46536262, -0.4670891 , -0.47663115]),\n",
       " 'split3_train_score': array([-0.4851387 , -0.48213504, -0.48084378, -0.4804802 , -0.48349703,\n",
       "        -0.47668791, -0.47548565, -0.47567053, -0.47637579, -0.48072488,\n",
       "        -0.47044879, -0.47038782, -0.47301301, -0.47457663, -0.48038772,\n",
       "        -0.46815121, -0.4678566 , -0.46783389, -0.47053499, -0.47769775,\n",
       "        -0.46631982, -0.46587683, -0.46754953, -0.4721118 , -0.47634923]),\n",
       " 'split4_train_score': array([-0.48224464, -0.4807065 , -0.48024049, -0.47995934, -0.48354135,\n",
       "        -0.47535375, -0.47388129, -0.47347412, -0.47563142, -0.4780774 ,\n",
       "        -0.47252098, -0.47138436, -0.47054112, -0.47136317, -0.48054289,\n",
       "        -0.47005499, -0.46911245, -0.46882929, -0.46897618, -0.47645396,\n",
       "        -0.46457073, -0.46480664, -0.46569152, -0.46794349, -0.47537336]),\n",
       " 'mean_train_score': array([-0.48362287, -0.481309  , -0.480874  , -0.48060048, -0.48370891,\n",
       "        -0.47706831, -0.47583578, -0.475547  , -0.4760472 , -0.47952071,\n",
       "        -0.47245583, -0.47101505, -0.47165084, -0.4734629 , -0.47973041,\n",
       "        -0.46916793, -0.46827484, -0.46866755, -0.47094618, -0.47671269,\n",
       "        -0.46633043, -0.46548207, -0.46685285, -0.46931005, -0.4757053 ]),\n",
       " 'std_train_score': array([0.00098827, 0.0007386 , 0.00104246, 0.00055312, 0.00048368,\n",
       "        0.00136422, 0.00144405, 0.00127751, 0.00076145, 0.0011156 ,\n",
       "        0.00117772, 0.00100083, 0.00089973, 0.00145693, 0.00164587,\n",
       "        0.00112641, 0.00088733, 0.00072959, 0.00126625, 0.00085395,\n",
       "        0.00161865, 0.00128964, 0.00130209, 0.00214443, 0.00179493])}"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch3_1.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.581020 using {'colsample_bytree': 0.6, 'subsample': 0.9}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAEKCAYAAAAvlUMdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3XdYVMcawOHf0EUUpYhYsZfQLKBo7GJJjFFjSGJBbLFcNdWrJteYHpN4k9hNYozGkhj1xpjYe0UFFWxYEFFRRAQbKH3uH2chiHRYlzLv8+yT3bNnzpndy/XbOWfm+4SUEkVRFEUpbkaG7oCiKIpSNqkAoyiKouiFCjCKoiiKXqgAoyiKouiFCjCKoiiKXqgAoyiKouiFCjCKoiiKXqgAoyiKouiFCjCKoiiKXpgYugOGZGdnJ52cnAzdDUVRlFLl2LFjt6WU9nntp9cAI4ToBcwGjIHFUsqZWd73A74Grus2zZNSLta99xXwPNooazvwhpRSCiH2AI7AI12bHlLKW0IIc+AXoBUQA7wipQzPrX9OTk4EBgYW9WMqiqKUK0KIK/nZT28BRghhDMwHvIEIIEAIsUFKeTbLrqullBOytG0HtAdcdZsOAJ2APbrXg6WUWSPDSOCOlLKhEOJV4EvgleL6PIqiKErB6PMejCcQKqUMk1ImAb8BL+azrQQsADPAHDAFovJo8yKwTPd8LdBNCCEK3GtFURSlWOgzwNQErmV6HaHbltVLQoiTQoi1QojaAFJKf2A3EKl7bJVShmRq87MQIkgIMT1TEMk4n5QyBbgH2GY9mRDidSFEoBAiMDo6uogfUVEURcmJPu/BZDd6yFob4C/gVyllohBiLNoIpKsQoiHQDKil22+7EKKjlHIf2uWx60KISsA6YCjavZf8nA8p5Q/ADwCtW7dWtQqUEiE5OZmIiAgSEhIM3RVFyWBhYUGtWrUwNTUtVHt9BpgIoHam17WAG5l3kFLGZHr5I9p9E4D+wGEpZRyAEGIz0BbYJ6W8rmv7QAixCu1S3C+ZzhchhDABrIHY4v5QiqIPERERVKpUCScnJ9SVXaUkkFISExNDREQE9erVK9Qx9HmJLABoJISoJ4QwA14FNmTeQQjhmOllXyD9MthVoJMQwkQIYYp2gz9E99pO19YU6AOc1rXZAAzTPR8I7JKqmppSSiQkJGBra6uCi1JiCCGwtbUt0qhabyMYKWWKEGICsBVtmvISKeUZIcTHQKCUcgMwSQjRF0hBG2346ZqvBboCp9Auc22RUv4lhKgIbNUFF2NgB9rIB+AnYLkQIlR3rFf19dkURR9UcFFKmqL+Tep1HYyUchOwKcu2DzI9nwZMy6ZdKjAmm+3xaOtcsjtXAvByEbucL+G341kdeI13ezTB2Ej9o6AoipIdlSqmELadvcnCPZeY+OtxElNSDd0dRVGUEkkFmEJ4vWMD/vN8MzaduonfkgAeJCQbukuKUiR3795lwYIFhWr73Xff8fDhw2Lry9KlS5kwYULeOxazPXv20KdPnwK3K2h/w8PDWbVqVYHPk1+xsbF4e3vTqFEjvL29uXPnTo773r9/n5o1a+rt+1YBppBGdajPt6+4ERAeyyvfH+bWAzW9VCm9SlKAKetyCzApKSlFPv7MmTPp1q0bFy9epFu3bsycOTPHfadPn06nTp2KfM6clOtkl0XVv0UtqlqaMW7FcQYu9Gf5SE/q2lY0dLeUUu6jv85w9sb9Yj1m8xqVmfHCMzm+P3XqVC5duoS7uzve3t5Uq1aN33//ncTERPr3789HH31EfHw8Pj4+REREkJqayvTp04mKiuLGjRt06dIFOzs7du/ene3xt2zZwnvvvUdqaip2dnbs3LmT2NhYRowYQVhYGJaWlvzwww+4uro+1m7NmjV89NFHGBsbY21tzb59+wgPD2fo0KHEx8cDMG/ePNq1a8eePXuYMWMGDg4OBAUFMWDAAFxcXJg9ezaPHj1i/fr1NGjQAD8/PywsLDhz5gxRUVF88803T4xc4uPjmThxIqdOnSIlJYUPP/yQF1/MORHJtWvX6NWrF5cvX2bQoEHMmDGD6dOnY2dnxxtvvAHA+++/j4ODA6tWrSIkJAR3d3eGDRtG1apV2bhxIwkJCcTHx7Nr1y6+/vrrJ75/gBUrVjBnzhySkpJo06YNCxYswNjY+LG+/Pnnn+zZsweAYcOG0blzZ7788kuyOnbsGFFRUfTq1UtvORlVgCmizk2qsWp0G0YsDeClhYdYOtwT55rWhu6WohTIzJkzOX36NEFBQWzbto21a9dy9OhRpJT07duXffv2ER0dTY0aNdi4cSMA9+7dw9ramm+++Ybdu3djZ2eX7bGjo6MZPXo0+/bto169esTGasvTZsyYQYsWLVi/fj27du3C19eXoKCgx9p+/PHHbN26lZo1a3L37l0AqlWrxvbt27GwsODixYu89tprGf9ABgcHExISgo2NDfXr12fUqFEcPXqU2bNnM3fuXL777jtAG0Xs3buXS5cu0aVLF0JDQx8772effUbXrl1ZsmQJd+/exdPTk+7du1OxYvY/II8ePcrp06extLTEw8OD559/npEjRzJgwADeeOMN0tLS+O233zh69Ciurq7MmjWLv//+G9Ausfn7+3Py5ElsbGzYtm0bFy9efOL7t7e3Z/Xq1Rw8eBBTU1PGjx/PypUr8fX1ZdSoUYwdO5bWrVsTFRWFo6O2AsTR0ZFbt2490d+0tDTeeecdli9fzs6dO/P+AykkFWCKQYs6VVkzth3Dlhzl1R8O88PQVrRrmP3/2RQlL7mNNJ6Gbdu2sW3bNlq0aAFAXFwcFy9epEOHDrz77rtMmTKFPn360KFDh3wd7/Dhw3Ts2DFjsZ6NjQ0ABw4cYN26dQB07dqVmJgY7t2791jb9u3b4+fnh4+PDwMGDAC0rAcTJkwgKCgIY2NjLly4kLG/h4dHxj+uDRo0oEePHgC4uLg8Nrry8fHByMiIRo0aUb9+fc6dO/fEd7BhwwZmzZoFaOuUrl69SrNmzbL9jN7e3tjaapmpBgwYwIEDB3jzzTextbXlxIkTREVF0aJFi4x9smuf/r3k9P2fPHmSY8eO4eHhAcCjR4+oVq0aAIsXL872uDlZsGABzz33HLVr18575yJQAaaYNKxmxbpxWpDx+zmAb19x53lXx7wbKkoJI6Vk2rRpjBnzxEoBjh07xqZNm5g2bRo9evTggw8+yOYITx4vu/UU2a2DzrrfokWLOHLkCBs3bsTd3Z2goCDmzp2Lg4MDwcHBpKWlYWFhkbG/ubl5xnMjI6OM10ZGRo/d38h6nqyvpZSsW7eOJk2a5Pn5cjveqFGjWLp0KTdv3mTEiBE5ts88Msrp+587dy7Dhg3jiy++yLUvDg4OREZG4ujoSGRkZEYQyszf35/9+/ezYMEC4uLiSEpKwsrKKtf7NYWhbvIXo+rWFvw+xgvXWtZM+PU4y/3DDd0lRcmXSpUq8eDBAwB69uzJkiVLiIuLA+D69evcunWLGzduYGlpyZAhQ3j33Xc5fvz4E22z4+Xlxd69e7l8+TJAxiWyjh07snLlSkCbwWVnZ0flypUfa3vp0iXatGnDxx9/jJ2dHdeuXePevXs4OjpiZGTE8uXLSU0t+FKBNWvWkJaWxqVLlwgLC3sikPTs2ZO5c+dmBMETJ07kerzt27cTGxubca+nffv2APTv358tW7YQEBBAz549gby/r5y+/27durF27dqMS16xsbFcufJkWZa+ffuybJmWWH7ZsmXZ3jtauXIlV69eJTw8nFmzZuHr61vswQXUCKbYWVuasmJUGyasOsH0P88Q/SCRt7wbq1XaSolma2tL+/btcXZ2pnfv3gwaNAgvLy8ArKysWLFiBaGhoUyePBkjIyNMTU1ZuHAhAK+//jq9e/fG0dEx25v89vb2/PDDDwwYMIC0tLSMeygffvghw4cPx9XVFUtLy4x/FDObPHkyFy9eREpJt27dcHNzY/z48bz00kusWbOGLl265HhfJDdNmjShU6dOREVFsWjRosdGQaDNrnrzzTdxdXVFSomTk1PGPZPsPPvsswwdOpTQ0FAGDRpE69atATAzM6NLly5UqVIl42a8q6srJiYmuLm54efnR9WqVR87Vo8ePQgJCXni+2/evDmffvopPXr0IC0tDVNTU+bPn0/dunUfuwczdepUfHx8+Omnn6hTpw5r1qwBIDAwkEWLFhX4clpRiPKcrqt169ZSX7MnUlLTeO+PU/weGMFrnnX4tJ+zWvWv5CgkJCTH6/tK8fLz86NPnz4MHDhQ7+dKS0ujZcuWrFmzhkaNGun9fPqQ3d+mEOKYlLJ1Xm3VJTI9MTE24suXXPlXlwb8evQq41ceIyFZrfpXlPLi7NmzNGzYkG7dupXa4FJU6hKZHgkhmNyzKXZW5nz011l8lxzlR9/WWFcoXG0FRSnp2rRpQ2Ji4mPbli9fjouLi4F69KSlS5cWqt3WrVuZMmXKY9vq1avHH3/8ke3+zZs3JywsrFDnKitUgHkKhrevh62VOe/8HsQr3/vzywhPqlW2yLuhopQyR44cMXQX9KZnz54ZN+qV/FGXyJ6Svm41WOLnwdXYhwxYeIjLt+MN3SVFURS9UgHmKerQyJ7fXm/Lw6RUBi48xMmIu4bukqIoit6oAPOUudaqwtqxXlQwM+a1Hw6z/2K0obukKIqiF3oNMEKIXkKI80KIUCHE1Gze9xNCRAshgnSPUZne+0oIcUYIESKEmCOyLCQRQmwQQpzO9PpDIcT1TMd6Tp+frSjq22ur/mvbWDJiaQAbgm8YukuKoijFTm8BRghhDMwHegPNgdeEEM2z2XW1lNJd91isa9sOaA+4As6AB5CRU1oIMQCIy+ZY32Y61qZs3i8xHCpbsHqMFy3qVGXSryf4+eBlQ3dJKcdKUrp+VQ+maPJbD8bY2Bh3d3fc3d3p27evXvqizxGMJxAqpQyTUiYBvwE557t+nAQsADPAHDAFogCEEFbA28Cnxd7jp8y6gim/jPCk5zMOfPTXWb7eei7b/EyKom8lKcCUdSWlHkyFChUICgoiKCiIDRs2FPm82dHnNOWawLVMryOANtns95IQoiNwAXhLSnlNSukvhNgNRAICmCelDNHt/wnwXyC7v+gJQghfIBB4R0qZcym3EsLC1JgFg1vxn/Wnmb/7ErcfJPFZf2dMjNXtsXJr81S4eap4j1ndBXrnnGtK1YMpf/VgngZ9Bpjs8qJk/Xn+F/CrlDJRCDEWWAZ0FUI0BJoBtXT7bdcFoftAQynlW0IIpyzHWogWfCT/BKEn0pcKIV4HXgeoU6dOIT5W8TM2Enze3xl7KzPm7AolJj6JeYNaYGFqnHdjRSkGqh5M+aoHA1oJgtatW2NiYsLUqVPp169f7n8khaDPABMBZC42UAt47G62lDIm08sfgfQw2x84LKWMAxBCbAbaAg+AVkKIcLS+VxNC7JFSdpZSRqUfSAjxI5BtZjop5Q/AD6DlIiv0pytmQgje7tEEu0rmzNhwhqE/HWGxrwfWlmrVf7mTy0jjaVD1YMp+PRiAq1evUqNGDcLCwujatSsuLi40aNCgwMfJjT4DTADQSAhRD7gOvAoMyryDEMJRShmpe9kXSL8MdhUYLYT4Am0k1An4Tkr5F9pIBd0I5m8pZedsjtUfyJhhVpr4ejlhW9Gct1YH4fO9P8tGeFLdWq36V54eVQ+m7NeDAahRowYA9evXp3Pnzpw4caLYA4zeLvRLKVOACcBWtMDxu5TyjBDiYyFE+pSFSbqpyMHAJMBPt30tcAk4BQQDwbrgkpuvhBCnhBAngS7AW8X7iZ6e510dWTrcg+t3H/HSwkOE3spuwpyiFB9VD6Z81YO5c+dORs6427dvc/DgQZo3z26Sb9HoNReZbqrwpizbPsj0fBowLZt2qcCTP58e3yccbQpz+uuhRexuidKuoR2/vd4Wv5+P8vKiQ/w83BP32lUM3S2ljFL1YMpXPZiQkBDGjBmDkZERaWlpTJ06VS8BRtWD0VM9mOISfjse3yVHiX6QyMIhLencJPvhrlK6qXowT4+qB1Mwqh5MGeZkV5G147yoZ1eRUcsCWX/iuqG7pChKPqh6MCpdf6lQrZIFq8e05fVfjvHm6iBuxyUyqkN9Q3dLUZ6g6sH8Q9WDUQGm1KhkYcrPwz14+/cgPt0YQnRcIlN7Nc12do6iGIqqB6NkpgJMKWJhaszc11piU/E03+8N4/aDJGa+5IKpWvWvKEoJpAJMKWNsJPjkRWfsrSz4dscF7jxMYv6gllQwU6v+FUUpWdRP31JICMEb3RvxWX9n9py/xeDFh7n7MMnQ3VIURXmMCjCl2OA2dVkwuCWnr99n4CJ/btx9ZOguKaVUScqmrNL1F01+0/VfvXqVHj160KxZM5o3b054eHix90UFmFKul7Mjy0Z4EnUvQbfqP+cVwoqSk5IUYMq6kpKu39fXl8mTJxMSEsLRo0dzTClTFCrAlAFeDWxZPcaLlDTJwEX+HLtS4qsUKCVM5nT9kydP5uuvv8bDwwNXV1dmzJgBaCnsn3/+edzc3HB2dmb16tXMmTMnI11/ly5dcjz+li1baNmyJW5ubnTr1g3Qfmn369cPV1dX2rZty8mTJ59ot2bNGpydnXFzc6Njx46A9g90hw4daNmyJS1btuTQoUOANgLp1KkTPj4+NG7cmKlTp7Jy5Uo8PT1xcXHh0qVLgLbQcuzYsXTo0IHGjRtnu0I/Pj6eESNG4OHhQYsWLfjzzz9z/f7S0/U3adIkI7X+9OnTmT17dsY+77//PnPmzGHq1Kns378fd3d3vv32W5YuXcrLL7/MCy+8kJGcM7vvH7R0/Z6enri7uzNmzJhs0+T8+eefDBs2DNDS9a9fv/6Jfc6ePUtKSgre3t6Ali3A0tIy189YGOomfxnRvEZl1o1th++SIwxefJgFg1vStamDobulFMKXR7/kXOy5vHcsgKY2TZniOSXH91W6/vKVrv/ChQtUqVKFAQMGcPnyZbp3787MmTOfqC1TVGoEU4bUsbVk7bh2NKpWidG/HGPdsQhDd0kphTKni2/ZsiXnzp3j4sWLuLi4sGPHDqZMmcL+/fuxtrbO1/FyS9c/dKiWQjCvdP0//vhjxq/15ORkRo8ejYuLCy+//DJnz57N2D89Xb+5ufkT6foz32PIT7r+mTNn4u7uTufOnTPS9eckPV1/hQoVMtL1Ozk5ZaTrT/8+C5quP/P3v3Pnzox0/e7u7uzcuTNjIefixYsz8p/lR0pKCvv372fWrFkEBAQQFhZW6AWouVEjmDLGzsqcX19vy5jlgbyzJpjbcYmM6VS8KbgV/cptpPE0qHT9ZT9df61atWjRogX162sZQfr168fhw4cZOXJk7h+0gNQIpgyyMjdhiZ8HfVwd+WLzOT7beJa0tPKb1FTJm0rXX77S9Xt4eHDnzh2io6MB2LVrV+lL168YjrmJMXNebYGdlTk/7r/M7bgkvhroqlb9K9lS6frLV7p+Y2NjZs2aRbdu3ZBS0qpVK0aPHl3g7zEvKl1/CU/XX1RSShbsucTXW8/TqbE9Cwa3pKK5+l1R0qh0/U+PStdfMCpdv5IjIQT/6tKQmQNc2H8xmkGLjxAbr1b9K4q+qXT9er5EJoToBcwGjIHFUsqZWd73A74G0ouczJNSLta99xXwPFoQ3A68ITMNt4QQG4D6Ukpn3WsbYDXgBIQDPlJKtSBE51XPOthUNGPirycYuOgQv4zwpFbV4p/3rpRvKl3/P1S6fj1eIhNCGAMXAG8gAggAXpNSns20jx/QWko5IUvbdmiBp6Nu0wFgmpRyj+79AcBAwDVTgPkKiJVSzhRCTAWqSilznY5THi6RZXX0ciwjlwVgaWbMLyPa0KR6JUN3SUFdIlNKrpJ6icwTCJVShkkpk4DfgCenM2RPAhaAGWAOmAJRAEIIK+Bt4NMsbV4E0u8SLgP6Fan3ZZRnPRvWjPVCSnh50SECwmMN3SVFUcoofQaYmsC1TK8jdNuyekkIcVIIsVYIURtASukP7AYidY+tUsoQ3f6fAP8FsiY/cpBSRuraRwLZJtYRQrwuhAgUQgSmT9Erb5pWr8y6ce2wszJnyOIjbD8bZeguKYpSBukzwGRXajHr9bi/ACcppSuwA90IRAjREGgG1EILSl2FEB2FEO5AQyll9hc980FK+YOUsrWUsrW9vX1hD1Pq1baxZM1YL5pWr8SY5YGsDsh5lbKiKEph6DPARAC1M72uBdzIvIOUMkZKmX5H8Eegle55f+CwlDJOShkHbAbaAl5AKyFEONp9mcZCiD26NlFCCEcA3X+fTMCjPMbWypxVo9vSvqEdU9adYv7u0GxXVyuKohSGPgNMANBICFFPCGEGvApsyLxDekDQ6QukXwa7CnQSQpgIIUyBTkCIlHKhlLKGlNIJeBa4IKXsrGuzARimez4MyD39qQJARXMTfhrmwYvuNfh663k++kut+i+PSlK6flUPpmjyUw9m9+7duLu7ZzwsLCyyzbpcVHoLMFLKFGACsBUtcPwupTwjhPhYCNFXt9skIcQZIUQwMAnw021fC1wCTgHBQLCU8q88TjkT8BZCXESbuZZ9EQTlCWYmRnzr486I9vVYeiicN1YHkZSSZuhuKU9RSQowZV1JqAfTpUsXgoKCCAoKYteuXVhaWmYkBi1Oel0HI6XcBGzKsu2DTM+nAdOyaZcKPJlp7/F9wgHnTK9jgG5F63H5ZWQkmN6nGfaVzPlyyznuxCexaGgrrNSq/3Ihcz0Yb29vqlWrxu+//05iYiL9+/fno48+Ij4+Hh8fHyIiIkhNTWX69OlERUVl1IOxs7PLNlUMaPVg3nvvPVJTU7Gzs2Pnzp3ExsYyYsQIwsLCsLS05IcffsDV1fWxdmvWrOGjjz7C2NgYa2tr9u3bR3h4OEOHDiU+Ph6AefPm0a5dO/bs2cOMGTNwcHAgKCiIAQMG4OLiwuzZszNyhDVo0AA/Pz8sLCw4c+YMUVFRfPPNN0+MXOLj45k4cSKnTp0iJSWFDz/8MNucXunS68FcvnyZQYMGMWPGDKZPn46dnR1vvPEGoNWDcXBwYNWqVYSEhODu7s6wYcOoWrUqGzduJCEhgfj4eHbt2sXXX3/9xPcPWj2YOXPmkJSURJs2bViwYMETKfb//PNP9uzZA2j1YDp37syXX36ZY9/Xrl1L7969VT0YRb+EEIzr3ABbKzOm/e8Ug348zBI/D+yszPNurBSbm59/TmJI8daDMW/WlOrvvZfj+6oeTPmqB5PZb7/9xttvv53rPoWlAozyBJ/WtbGtaMa/Vh3n5UX+/DLCk9o2atV/eZG5HglAXFwcFy9epEOHDrz77rtMmTKFPn360KFDh3wdL7d6MOvWrQPyrgfj4+PDgAEDAK0ezIQJEwgKCsLY2JgLFy5k7J9eDwZ4oh5M5tFVfurBbNiwgVmzZgFk1IPJaTFsej0YIKMezJtvvplRDyYqKqpQ9WDgn+//5MmTGfVgAB49epSRin/x4sXZHjcvkZGRnDp1KiPTc3FTAUbJVrdmDqwc1YYRSwMZsPAQy4Z70rxG5bwbKkWW20jjaVD1YMp+PZh0v//+O/3798fU1DTXYxaWSnap5KhVXW3Vv7EQvPK9P4fDYgzdJUVPVD2Y8lUPJt2vv/7Ka6+9lutnKwo1glFy1dihEuvGt8P3pyP4LjnKnFdb0Mu5uqG7pRQzVQ+mfNWDAe0+1LVr1+jUqVOBv7/8UvVgylmyy8K6E5/E8KUBnIy4y6f9XBjUpo6hu1SmqGSXT4+qB1MwJTXZpVKGVK1oxqrRbejY2J73/jjFnJ0X1ap/RcmFqgejLpEpBWBpZsKPvq2ZsvYk32y/wO24RGa88AzGRtmlnVPKI1UP5h+qHowKMEoBmRobMetlN+wqmfPDvjBi4pL45hU3zE2M826slHlHjhwxdBf0pmfPnnqbzltWqQCjFJiRkeC955phZ2XG55vOcedhEt8PbUUlC/1MdVQUpXRS92CUQnu9YwO+8XHj6OVYXv3hMNEPEvNupChKuaECjFIkA1rW4sdhrQmLjmfgokNciYk3dJcURSkhVIBRiqxLk2qsHN2Ge4+SeWmhP6ev38u7kaIoZZ4KMEqxaFmnKmvHemFmLHj1h8McunTb0F1SCqAkpetX9WCKJj/1YAD+/e9/88wzz9CsWTMmTZqkl2UHKsAUQkpaCsHRwYbuRonTsJq26t/R2gK/JQFsOhVp6C4p+VSSAkxZVxLqwRw6dIiDBw9y8uRJTp8+TUBAAHv37i3yubNSs8gKYUHQAn4+8zOLui+ijWMbQ3enRHG0rsCasV6MXBbIv1Yd5+MXnRnatq6hu1Wq7P/9ArevxRXrMe1qW9HBp3GO76t6MOWrHowQgoSEBJKSkpBSkpycjIODQ46fr7D0OoIRQvQSQpwXQoQKIaZm876fECJaCBGke4zK9N5XumqXIUKIOUKXnlQIsUUIEax7b5EQwli3/UMhxPVMx3pOX59r2DPDcKrsxJu73+R87Hl9nabUqmJpxoqRbejapBrT15/mm+0X1Kr/Em7mzJk0aNCAoKAgvL29M+qRBAUFcezYMfbt28eWLVuoUaMGwcHBnD59ml69ejFp0iRq1KjB7t27cwwu6fVg1q1bR3BwcEZurPR6MCdPnuTzzz/H19f3ibbp9WCCg4PZsEGruJ6ey+z48eOsXr2aSZMmZewfHBzM7NmzOXXqFMuXL+fChQscPXqUUaNGMXfu3Iz90uvBbNy4kbFjx5KQkPDYedPrwQQEBLB7924mT56cEdCyc/ToUVauXElQUBBr1qwhMDCQkSNHZuRXS68HM3jwYGbOnEmHDh0ICgrirbfeAsDf359ly5axa9eux+rBZP7+Q0JCMurBpJcqSE8WOmrUqIyaOPmpB+Pl5UWXLl1wdHTE0dGRnj176iVVkd5GMLp/+OejlS+OAAKEEBuklGez7LpaSjkhS9t2QHsg/efMAaATsAfwkVLe1wWctcDLwG+6/b6VUs7Sx+fJzNrcmoXdFzJ402DG7RjHiudWUMOqhr5PW6pUMDPm+6GtmPY/La1M9INEZrzQHAtTtSAzL7mNNJ6YzheGAAAgAElEQVQGVQ+m7NeDCQ0NJSQkhIiIiIzz79u3j44dOxboOHnR5yUyTyBUShkGIIT4DXgRyBpgsiMBC8AMEIApEAUgpbyv28dE975BfhpXr1idhd0X4rfZj3E7xvFL71+wNrc2RFdKLBNjI74a6IpdJXMW7rmE/6XbfNLPmQ6N7A3dNSUXqh5M2a8H88cff9C2bVusrKwA6N27d8YPgeKkz0tkNYFrmV5H6LZl9ZIQ4qQQYq0QojaAlNIf2A1E6h5bpZQh6Q2EEFuBW8ADtFFMugm6Yy0RQjyeA1sPGldtzOyus7n24BoTd00kISUh70bljBCCKb2asnykJ0IIhv50lH+tOk7UffVdlSSqHkz5qgdTp04d9u7dS0pKCsnJyezdu1cvl8j0GWCyy4CY9SfLX4CTlNIV2AEsAxBCNASaAbXQglJXIURGaJVS9gQcAXOgq27zQqAB4I4WlP6bbaeEeF0IESiECIyOji7kR/uHR3UPPu/wOUG3gpi6fyqpaQX/Yy8POjSyZ/MbHXire2O2n42i23/3suTAZVJS0wzdNYXH68Fs3749ox6Mi4sLAwcO5MGDB5w6dQpPT0/c3d357LPP+M9//gP8Uw+mS5cu2R47cz0YNzc3XnnlFQA+/PBDAgMDcXV1ZerUqTnWg3FxccHZ2ZmOHTtm1INZtmwZbdu25cKFC0WqB9O7d+8c68EkJyfj6uqKs7Mz06dPz/V46fVg3N3deemll56oB+Pj45NtPZhvv/32iWP16NEj2+8/cz0YV1dXvL29iYzUZmpmvgczdepUtm/fTqNGjdi+fTtTp2q3vwMDAxk1SrvNPXDgQBo0aICLiwtubm64ubnxwgsvFPh7zJOUUi8PwAtt5JH+ehowLZf9jYF7uueTgemZ3vsA+Hc2bYYB87LZ7gSczquPrVq1ksVl+Znl0nmps/zE/xOZlpZWbMctiy5Hx8mhPx2Rdaf8LZ+bvU8evxJr6C4Z3NmzZw3dhXJj2LBhcs2aNU/lXKmpqdLNzU1euHDhqZxPH7L72wQCZT7igD5HMAFAIyFEPSGEGfAqsCHzDkIIx0wv+wLpl8GuAp2EECZCCFO0G/whQgir9DZCCBPgOeBcNsfqD5zWw2fK0ZDmQ/B7xo/V51fz0+mfnuapSx0nu4osG+7B/EEtuR2XyICFh3j/j1Pce5hs6K4pSrFR9WD0eJNfSpkihJgAbEUbnSyRUp4RQnyMFv02AJOEEH2BFCAW8NM1X4t26esU2mW1LVLKv4QQDsAGIYS57pi7gEW6Nl8JIdx1+4cDT96h1LO3Wr3FrYe3mH18NvYV7HmxYc7z5ss7IQTPuzrSsbEd326/yNJDl9l65ibvPdeM/i1qZntTWCn5VD2Yf6h6MKpkcrGXTE5OTWbcznEE3gxkXrd5PFvz2WI9fll15sY9/rP+NCeu3qVtfRs+7edMw2qVDN2tp0aVTFZKKlUyuQQxNTblu87f0bBKQ97e8zZnYs4YukulwjM1rFk3th2f93chJPIBvWfv56st53iUVH4mTZTnH3tKyVTUv0kVYPTAysyKBd0XUNW8KuN3jOfa/Wt5N1IwMhIMalOHne90oq9bTRbsuUT3b/ayMyTK0F3TOwsLC2JiYlSQUUoMKSUxMTFPzLArCHWJrJgvkWUWdi8M382+WJtZs/y55dhY2OjtXGXRkbAY/rP+NBdvxdGjuQMz+j5DzSoVDN0tvUhOTiYiIuKJlCWKYkgWFhbUqlULU9PHq9Xm9xJZngFGCNEAiJBSJgohOqOlb/lFSnm38N0uGfQdYACCbgUxatsoGldtzOIei7E0tdTr+cqapJQ0fjpwmTk7LwLwRvdGjHy2HqbGavCtKIZSnPdg1gGpusWPPwH1AP0VMyhj3Ku582XHLzkTc4bJ+yaTklb0dNzliZmJEeM6N2D72x1p39COmZvP8fyc/Ry9HGvorimKkof8BJg0KWUK2tqS76SUb6GtolfyqVudbrzf5n32Rezjk8OfqOvshVCrqiWLh7XmR9/WxCem4vO9P++uCSYmLjHvxoqiGER+1sEkCyFeQ1s1n55LwDSX/ZVs+DTxIephFD+c/AEHSwfGu483dJdKJe/mDrRvaMucnaEs3h/GjpAopvRqyiuta2NkpNbOKEpJkp8RzHC0tC+fSSkvCyHqASv0262yaYL7BPo17MfC4IWsubDG0N0ptSzNTJjauymb3uhAY4dKTPvfKQYuOsTZG/fzbqwoylNToFlkugzFtaWUJ/XXpafnadzkzyo5LZlJuyZx6MYhZneZTefanZ/q+csaKSX/O36dzzaFcO9RMn7tnHjLuzFW5qpYq6LoS7Hd5BdC7BFCVBZC2ADBwM9CiG+Ko5PlkamRKf/t9F+a2TRj8t7JBEcHG7pLpZoQgpda1WLXO514xaM2Sw5epvt/97LpVKS616UoBpafS2TWUivyNQD4WUrZCuiu326VbZamlszvNh97S3sm7JzA5XuXDd2lUq+KpRmf93dh3bh22FQ0Y/zK4/j9HMCVmJzL3CqKol/5CTAmukzFPsDfeu5PuWFbwZZF3RdhJIwYt2Mctx/dNnSXyoSWdaqyYUJ7pvdpTmB4LD2+3cecnRdJTCk/KWcUpaTIT4D5GC0j8iUpZYAQoj5wUb/dKh/qVK7D/G7ziU2IZfyO8cQlxRm6S2WCibERI5+tx853OtO9mQPfbL9A7+/2czBUBXFFeZpUqpinfJM/O/sj9jNx10Q8qnuwoNsCTI3VLPDitPdCNB/8eZorMQ/p61aD//RpRrVKhc+vpCjlXXHe5K8lhPhDCHFLCBElhFgnhKhVPN1UADrU6sAMrxkcjjzMB4c+UDeni1mnxvZsfbMjk7o1Ysvpm3SbtZdlh8JJTVPfs6LoU34ukf2MVomyBlAT+Eu3TSlG/Rv1Z4L7BP4O+5vvjn9n6O6UORamxrzt3Zgtb3bArXYVZmw4Q7/5BzkZUepT6ilKiZWfAGMvpfxZSpmieywF7PXcr3LpddfX8Wnsw5LTS1gZstLQ3SmT6ttbsXykJ3Nfa0HU/QRenH+Q6etPc++RKtesKMUtPwHmthBiiBDCWPcYAsTk5+BCiF5CiPNCiFAhxNRs3vcTQkQLIYJ0j1GZ3vtKCHFGCBEihJgjdDV0hRBbhBDBuvcWCSGMddtthBDbhRAXdf+tmr+voOQQQvBem/foUrsLXx79ku1Xthu6S2WSEIIX3Gqw451ODPNyYuWRK3T7717Wn7iuLk8qSjHKT4AZgTZF+SYQCQxESx+TK90//POB3kBz4DUhRPNsdl0tpXTXPRbr2rYD2qOVBnAGPIBOuv19pJRuuu32wMu67VOBnVLKRsBO3etSx9jImC87fomrvStT903lWNQxQ3epzKpsYcqHfZ9hw4RnqVm1Am+uDmLw4iOE3lKz+RSlOOQZYKSUV6WUfaWU9lLKalLKfmiLLvPiCYRKKcOklEnAb8CL+eyXBCwAM8AcLblmlK4/6QmnTHTvp//kfBFYpnu+DOiXz3OVOBVMKjCv6zxqWNVg4q6JhN4JNXSXyjTnmtb8b1w7Pu3nzOnr9+g9ex+ztp4nIVmtnVGUoihs1aa387FPTSBzreAI3basXhJCnBRCrBVC1AaQUvoDu9FGTJHAVillSHoDIcRW4BbwAFir2+wgpYzUtY8EqhXsIxXAuY2w6hU4vhzi9bO2oopFFRZ5L8Lc2JyxO8ZyM/6mXs6jaIyNBEPa1mXnO515wbUG83aH4v3tXnafu2XorilKqVXYAJOfvOjZ7ZP1AvdfgJOU0hXYgW4Eoitu1gyohRaUugohOmYcRMqeaDVpzIGuBeq4EK8LIQKFEIHR0dEFafqPxAcQdQY2TIBZjWBJb/CfD7HFm/KlplVNFnZfSFxyHON2jON+ksoWrG/2lcz55hV3fh3dFjNjI4YvDWDs8mNE3ntk6K4pSqlTqIWWQoirUso6eezjBXyoCwYIIaYBSCm/yGF/YyBWSmkthJgMWEgpP9G99wGQIKX8KkubYYCHlHKCEOI80FlKGalLbbNHStkktz4WaaGllHDzpDaaCfkbbp3Rtjs4Q9PntUd1VxBFr1FyOPIw43aMw93ene+9v8fM2KzIx1TylpSSxo/7w5i76yJGQvBW98b4tXdS5ZqVcq/ICy2FEA+EEPezeTxAWxOTlwCgkRCinhDCDHgVbT1N5nNkrozZF0i/DHYV6CSEMBFCmKLd4A8RQliltxFCmADPAed0bTagFUVD998/89HHwhMCHN2gy3sw/hBMOgE9PgPzyrD3K/i+I3znCpunwuX9kFr4UsltHdvyaftPCYwK5L0D75Em04rxgyg5MTMx4l9dGrL9rU541bfls00hvDD3AMeuqHLNipIfek0VI4R4DvgOMAaWSCk/E0J8DARKKTcIIb5ACywpQCwwTkp5TjeaWQB0RLustkVK+bYQwgEt4aa57pi7gLeklClCCFvgd6AOWoB6WUqZ678EeksVExcNFzZro5tLuyE1ESrYQJPe2simfhcwsyzwYZeeXsp/j/2XIc2G8G+PfyOKYXSk5I+Ukm1no/hwwxki7yXwSuvaTO3dlKoV1WhSKX/yO4JRucj0nYssMQ4u7dSCzYUtkHAPTCpAw25asGncCyxt8nUoKSVfBXzFipAVvNv6XYY9MyzvRkqxik9MYc7Oi/x04DKVLEyY1rsZA1vVUuWalXJFBZh8eOrJLlOTIfyAFmzObYQHN0AYQ9120LQPNH0OquR6a4s0mcbkvZPZdmUbMzvM5Pn6zz+lziuZnb/5gP+sP0VA+B1a163Kp/2daVq9sqG7pShPhQow+WDQbMpSwo3j/wSbaN2tpOqu0OwFbXRTrXm2kwQSUxMZu30sQdFBLOy+kLaObZ9y5xWAtDTJ2uMRfLEphPsJKYx8th5vdGtERVWuWSnjVIDJh5KSrh+A26FwXjcjLSIAkFDVSTey6QO1PcHIOGP3+0n3GbZ5GJHxkSzttZSmNk0N1vXy7k58El9uOcdvAdeoYW3BBy88Q89nHNQ9MqXMKrYAo5s1lnWne0Ag8I6UMqzQvTSwEhVgMntwE87rJglc3gupSWBpp5sk0AfqdwZTC27G32TIpiGkylRWPLeCmlbZrWNVnpZjV2J5/4/TnLv5gK5Nq/FR32eobVPwyRyKUtIVZ4D5CLgBrEJbPPkqUB04jzbrq3ORe2sgJTbAZJZwH0K36yYJbIOkB2BaERp1h6Z9CK3WEN/dE7C1sGV57+VUsahi6B6XaympaSw9FM432y+QJiUTuzZidIf6mJmotTNK2VGcAeaIlLJNlm2HpZRthRDBusSTpVKpCDCZpSRC+H7tMtr5TRAXBUYmBNZtxesiimeqNuXH3suwMFHVGg0t8t4jPv7rLJtP36SBfUU+6edMuwZ2hu6WohSLYqtoCaQJIXyEEEa6h0+m98rvDRxDMDGHht3hhe/g7XMwcgd4TaD1/Rhm3owiOOYM//6lPal7v4Jb57SJBIpBOFpXYOGQVvw83IOk1DQG/XiEt1YHEf0g0dBdU5SnJj8jmPrAbMBLt8kfeAu4DrSSUh7Qaw/1qNSNYHITfZ6VR75mZswRXrn/gPdj7iBsG+rS1vSBmq3BSF2mMYSE5FTm7w5l0d5LWJga8+9eTRnkWQdjtXZGKaXULLJ8KFMBRuebY9/w8+mfmWjvxeu3b2mX1NJSwMpBN0ngBajXQRsNKU/Vpeg4pq8/zaFLMbjVsuaz/i4417Q2dLcUpcCK8x5MLWAuWgEwCRwA3pBSRhRHRw2pLAaYNJnG+wfe5++wv/mk/Sf0q9kZLm6Hc39r/02OB7NK0MhbG9008gYL9Y/c0yKlZEPwDT75O4TY+ER8vZx4u0djKluYGrpripJvxRlgtqPNIFuu2zQEGCyl9C5yLw2sLAYYgOTUZMbvHE/AzQDmdp1Lh1oddG8kaNOez/2tTYOOjwYjU6jfSQs2TZ6DStUN2/ly4t6jZP677TzLD1/Bzsqc6X2a84Kro1o7o5QKxRlggqSU7nltK43KaoABiEuKY/jW4Vy5f4UlPZfgbOf8+A5pqdqCznN/a7PS7uhq2dTy+Gdxp13Dp9/xcuZkxF3e/+M0p67f49mGdnz84jPUt7cydLcUJVfFGWB2AEuBX3WbXgOGSym7FbWThlaWAwzA7Ue3GbJpCI9SHrGi9wpqV66d/Y5Swq0QXdqavyEySNtu1+SfSQI1WqhJAnqSmiZZeeQKX285T2JKGmM7N2B85wZYmBrn3VhRDKA4A0wdYB7aLDIJHAImSSmvFkdHDamsBxiAy/cuM3TzUCqbVWZ57+XYVrDNu9Hda9o6m3N/Q/hBkKlQqYaWjLPp81D3WTBRaeqL260HCXy2MYQ/g25Qx8aS0R3q0b9lLaxUbjOlhNHrLDIhxJtSyu8K1bMSpDwEGICgW0GM3jaaBlUasKTnEixNC5C+5GEsXNwGIX9B6E5IeQTm1tC4pxZsGnYHc3VJpzgdDL3NzM3nOHX9HlbmJrzUsiZDverSsFolQ3dNUQD9B5g8SyaXBuUlwADsvrqbN/e8Sbsa7ZjTdQ6mRoWYtZT0EML2aJfSzm+CR7FgbK7lRmvWBxr3Biv7Yu55+SSl5MS1uyz3v8LGk5EkpabRroEtvl516d7MARNVtlkxIH0HmGtSyhwu6Jce5SnAAKy5sIaP/T+mf8P+fNTuo6LNWEpNgWuH/7lvc/cqIKBOW919m+fBpn6x9b08ux2XyOqAa6w6cpXrdx/haG3BIM86vOpZB/tKaj2T8vSViBGMEKIXWhYAY2CxlHJmlvf9gK/RsgIAzJNSLta99xXwPFo6m+3AG0AFYA3QAEgF/pJSTs3rWDkpbwEGYH7QfBYFL2KM6xgmtJhQPAeVEqJOa8Em5G+IOqVtr/bMP8HG0S3b2jZK/qWmSXaGRLH88BX2X7yNqbGgt7Mjvl51aVW3qprirDw1RQ4wOaTpBy2jcgUpZa53HoUQxsAFwBuIAAKA16SUZzPt4we0llJOyNK2HVqw6KjbdACYBhwF2kgpdwshzICdwOdSys05HSs35THASCmZcWgGf4T+wfS20/Fp4pN3o4K6Ew7nNmkB5+ohkGlQ0R4q2IBpBTCrCKaWjz83s9Rts9Rtq5DlecUs+1hqpafL6cy2S9FxrDh8hbXHIniQkEJzx8oM9arLi+41sDRTkwIU/cpvgMnxL1FKWdQ7ip5AaHq9GCHEb8CLwNlcW+lOD1gAZmgBzRSIklI+BHbr+pckhDgO1CpiP8sVIQTTvaZz+9FtPjvyGXYV7Ohap2vxnqSqE3iN1x7xMXBhM1zx10oNJD2E5Ifw8PY/z5Mfas9THhX8XCYVdIFHF4jSn5tZZglMOQSprO2yBsASOipoYG/FjBeeYXLPJqw/cYNf/MOZ9r9TfLEphJdb12ZI27rUs6to6G4q5ZzecpEJIQYCvaSUo3Svh6KNPiZk2scP+AKIRhvtvCWlvKZ7bxYwCi3AzJNSvp/l+FWA40B3KWVYbsfK0u514HWAOnXqtLpy5UpxfuxS42HyQ0ZuHcnFuxdZ3GMx7tVKwLrZtDRdwHmkpbRJyvo8UzBKjtfeS4r/p0368yf21T1SEgrep8eCUW4jLMvsR2ZPtM0UAE0sii2ASSkJCL/DL/7hbDl9k5Q0ScfG9vi2rUuXptVUYk2lWBk82aUQ4mWgZ5YA4ymlnJhpH1sgTkqZKIQYC/hIKbsKIRqi3bt5RbfrdmCKlHKfrp0J8BewNX26dE7Hyq2Phb1Elnr/PilRUZg3alTgtiVJzKMYfDf7ci/pHst7L6eedT1Dd0m/0lJ1Aeth9oEpp+CUFJ+/oJda0FT8IlPQyTLCqlAV3F6Dxr0KfBnw1v0Efj16jVVHrxB1P5FaVSswuE1dXvGojU1FtX5JKbqSEGC8gA+llD11r6cBSCm/yGF/YyBWSmkthJgMWEgpP9G99wGQIKX8Svd6CVowmZTXsXLrY2EDzK0584he9D1VevfE7l//wrx+6f2H+dr9awzZPAQLYwtWPLcCe0s1zbjQUlO0y3yPjbAe5jACy2M0diccHkSCXWNoNxFcXylwBuzk1DS2n43iF/9wDofFYmZiRB9XR3y9nHCvrSqfKoVXEgKMCdqlqm5oM7sCgEFSyjOZ9nGUUkbqnvdHG6W0FUK8AowGeqFdItsCfCel/EsI8SnQDHhZSpmW17Fy62NhA8zZnZcIWH+Bemd+w+5mIFX69sVu/DjM6pTOpUFnbp9h+Nbh1K1cl597/oyVmVo4aXCpKXB2PRz8Dm6e0sottBkLrUdAhYIHhwtRD1juf4X/HY8gPikV11rW+Ho50cfVUaWkUQrM4AFG14nngO/QpikvkVJ+JoT4GAiUUm4QQnwB9AVSgFhgnJTynG4EsgBtFpkEtkgp39aVDrgGnAPSr0fMk1IuzulYufWvsAEm4lws+3+/SOyNeGzNH1AvcAmV74VRpX8/7MaNw7RGjQIf09AOXD/AxJ0TaVW9FQu7LcTUWKWPLxGk1Ba3HpwNYbvBzApa+UHbcWBd8PktDxKS+ePEdX7xv0LorTiqWpri41GbIW3qUtumABkelHKtRASYkq4o05TTUtM453+TIxvCeHg/iZoVblPnwEIqJMRQ9eWB2I4Zg6mDQzH3WL/+DP2T/xz8D8/Xf57Pn/0cI1E+pwCXWJEn4dAcOP0/bXKA80Dt8ll157zbZiGlxP9SDL/4X2F7SBRpUtK1STWGetWlYyN7jNSkACUXKsDkQ3Gsg0lKSCFo+1VObL9KWkoa9c0jcNw9H7O0BKq+9iq2o0djYmdXTD3Wvx9P/sicE3MY7jyct1u9bejuKNm5exX8F8DxX7T7Nw27Q7tJUK9joWalRd57xKojV/n16DVuxyXiZGvJkLZ1eblVbawt1UhWeZIKMPlQnAst4+8mcvSvMEIORWJqbkRjcQ67bQsxNjPGZvAgbEaOxKRq1WI5lz5JKfnsyGesPr+aqZ5TGdxssKG7pOTkYSwE/gRHvteKxzm6Q/tJ0OxFMC74YsuklDQ2n45kuf8VAq/cwcLUiBfdtESbqrSzkpkKMPmgj5X8MdfjOPS/S1w9E4OVtQlNEwOptHUxxhUqUHWYL7bDh2NcuXKxnrO4paal8vaet9l9bTdfd/qank49Dd0lJTfJCRD8K/jPg5hQqFIXvCZAi8HaepxCOHPjHisOX2H9iRs8Sk6lVd2q+HrVpbezI2Ym6tJpeacCTD7oM1XMtbOxHPxfKDERcdhXN6NJ7G7Mtq3CqFIlbIb7YePri7FVyZ2tlZCSwOhtozkTc4bvvb/Ho7qHobuk5CUtVctyfXAORBzVUvN4jgbP16Fi4S7T3nuUzNpjESz3Dyc85iF2Vma86lGHQW3qUKNKheLtv1JqqACTD/rORZaWJrlw5CaH/wwj/m4idRtY0ODyBtj1J8bW1tiMGonN4MEYWZbM2Tv3Eu8xdPNQbj+8zbLey2hUtXQvLC1Xrh7WZp6d36RlDHAfDF7/AtsGhTpcWppkf+htlvuHs/PcLYyEoHuzavh6OdGuga1KtFnOqACTD08r2WVyUirBO65xfOsVUpPTaPKMBbWDV5GybwfGNjbYjh5N1ddexcjCQu99KagbcTcYsmkIQghWPreS6hWrG7pLSkFEn4dDc+HkakhLgWYvQLs3oFarQh/yWuxDVh65yuqAq9x5mEwD+4r4ejkxoGVNKlmoSQHlgQow+fC0syk/vJ9EwN+XOXPgBqZmRri4mlFt/08k+h/ExN4e2zFjqOLzMkZmJSudx/nY8/ht8aN6xeos7bUUa3N1w7fUeXATjiyCgCWQeE8re91+EjT0LnRG6oTkVDaejOSXw1cIvnaXimbG9G9ZE18vJxo7qOqbZZkKMPlgqHT9d27Gc+h/lwg/eRurqua0cIFKmxaSEHgME0dH7MaOpcqA/gjTkvNr8EjkEcbuGIubvRvfe3+PubEqdFUqJT6AY8vg8AK4fx3sm2mBxnkgmBT+h83JiLv84n+FDcE3SEpJo009G3y9nOjxjAOmqvpmmaMCTD4Yuh7M9fN3OLgulOirD7CvbUWLJomYrJlHQvBJTGvVwm78eKz7voAwKRn1PTaFbWLK/il41/VmVqdZaiFmaZaaDKfXaRMCbp2BSjW07ACt/MCi8LMcY+OT+D3wGisOXyHiziMcKpvzmmcdBnnWoVrlkncJWCkcFWDywdABBkCmSS4GRuG//hJxsYnUdbbFrXYsKcvmkXD2LGZOTtj9619Ufq43wtjwOaOWnVnGrMBZDG42mCkeU9TN3dJOSgjdCYdmw+V9YF4ZWg+HNuOgsmOhD5uaJtlz/ha/+F9h74VoTIwEPZ2r49u2Lp71bNTfTSmnAkw+lIQAky4lOZWTuyI4tjmc5MRUmj1bg+aVrxH/4zwSL1zArGED7CdMpFIPb4SBqzh+FfAVy88u5+1WbzPcebhB+6IUoxsntBHN2fUgjMHVR0tFU61ZkQ4bfjueFYev8HvgNe4npNC0eiWGetWln3tNKpqXjNG5UjAqwORDSQow6R7FJRG4MZzTe69jZGpES+/a1JfnubtwHklhYZg3bYr9pIlYdelisF+BaTKNf+/7N1vDt/JFhy/oU7+PQfqh6EnsZe0ezfHlWvmBRj2h/RtQt12RCqQ9SkplQ7CWaPPMjftUMjfhpVa1GOpVlwb2JXdNmPIkFWDyoSQGmHR3ox5yeP0lLp2IpqK1GZ59nHCMPU7MgvkkX7mKhYsL9pMmUvHZZw0SaJJSkxi7Yywnbp1gQbcFeNXweup9UPQsPgYCFsPR7+FhDNRspQWapn3AqPCXa6WUHL96l+X+4Ww8FUlyquTZhnYM9apLt6bVMFGTAko8FWDyoSQHmHSRoXc5uC6UqMv3sa1ZEa9+9bC+sJ/b8xeQfOMGFVq0wH7SRCzbtn3qgeZ+0n38tvhx/cF1lvZaSjPbosUzJ0oAACAASURBVF1KUUqopIcQvAoOzYM7l8GmvpaKxn2QVomzCKIfJLI64Corj1wl8l4CNawtGNxWq75pZ6VmKpZUKsDkQ2kIMKD94rt0PBr/P0K5fzuB2s1t8HqhLiZHtnJ74SJSoqKw9PDA/o1JWLbO83/zYhUVH8WQzUNISUthxXMrqGlV86meX3mK0lIh5C8tQ8CN42BpB23GgMcosLQp0qFTUtPYEXKL5YfDORgag5mxEc+5VGeolxMt61RRkwJKGBVg8qG0BJh0qclpnNobQeCmcBIfpfD/9s48vqrq3PvfdaaczAkkTAmEGZlnCKigUBXnOlyHvqJcp6pFrdbh+mp7fbXWXmyvirT1VutEW71ttRVbZ1BRIQNDkDAnQMhI5jk54/P+sXfCISRwEnJIIOv7+eRz9rDW3s/e2Wf/zlrrWc8zft5gZl2UhPeT9yn//e/xlZcTOX8+ifffR/jUqafMrtzqXJZ+tJT+zv6svng1cU6djveMRgTyvjUcAvZ9AvYImL4U5t0D8cNP+vA5pfX8MS2PdzcXUOfyMnFIDLfMG87lU4cQ7uh5T0pNLxEYpdQS4EWMjJavisgv2+xfBjyHkVIZzOyU5r4VwKWABfgMuB8IB/4KjAJ8wAci8h9m+TDgLWAmUAFcLyIHj2ff6SYwLTQ3eNj80UG++7IAi1JMu2AYU88dQMN7f6Hi1VfxVVURtXAhCffdS/jEiafEps2HN3Pnp3cyvv94XrnwFcJtOhBin6B0lxmK5i8gPph4lZGbZsi0kz50g8vL37cWsnpjHnsO1xEbbue6WcnclJpCSv+uRYnWdA89LjBm2uO9wAVAAZAJ3CgiOwPKLANmicjyNnXnYwjPAnPTN8BjQAYwV0S+UEo5gLXAL0TkI6XUPcAUEblLKXUDcJWIXH88G09XgWmhpqyJtPdzydlUSniMgzmXjWDc1Fhq3v4TFa+/jr+mhugLvkfC8ntxjhsbcns+z/ucB798kIVDF/L8ec9js2gX1D5DbRGk/Q42vQ7uOhix0IgQMGrxSXmegdFFnHGgkrfS8vgkuwSfCAvHJnLzvBQWjh2AVWffPOX0BoGZBzwpIheZ648BiMizAWWW0b7AzANWAecAClgPLBWRXW3KvQhki8grSqlPzPNtVErZgBIgUY5zgae7wLRQcqCGDe/mUJxTQ/ygCOZfM5rkFAdVb62m8o038Dc0EHPxEhKWLyds5MiQ2vLnXX/m2YxnuXbstfws9We677yv0VwDm98wxKauGAZOMlo0k64G68mHPjpc22xm3zxEaZ2Lof3CuWluCtfNGkp8ZO+K4Xcm0xsE5lpgiYjcbq4vxWh9LA8oswx4FijDaO08ICL55r5fAbdjCMwqEXm8zfHjgC3A90Rkv1Iq2zxfgbk/1zxfeUc2nikCA8avvAPbytnwXg41pU0kjYvj7GvG0C/GR8Xrb1C5ejXS3Ezs5ZeRcM89OFJSQmbLC5tf4A/Zf2D5tOX8cOoPQ3YeTS/G64btf4UNK6FsN8QkG2M0M26GsJMPhOnx+flkRwlvbcwj40AlDpuFKUmxTB0axzTzLzk+XP/ACRG9QWD+DbiojcDMEZF7A8r0B+pFxKWUugu4TkQWKaVGY4zdtHRxfQY8KiLrzXo24APgExF5wdy2wzxfoMDMEZGKNnbdCdwJMGzYsJl5eXkhuf6ewufzs2N9EZn/PEBzg4dxcwcx98qRhNNIxat/oOrPf0Y8HmK/fyUJd9+DI7n7vb5EhMe/eZwP9n/AU/Of4qoxV3X7OTSnCX4/5HxmOATkfQPOWJh1G8y9C6IHdsspdpfU8u7mArYcqia7sAaX1w9A/0gHU4fGMTU5jqlDY5k2NI64CN3K6Q56g8CcsIusTXkrUCkisUqphwGniDxt7vsZ0CwiK8z11zCE6b6A+n22i6w9XE1etnycx7a1+QBMXTyUGUtSsNRVUvHKq1S/8w4CxF1zNQl33YV9UPfmefH4PSxfu5z04nRWLlrJguQFJ66kObMp2GzEPNu5xugum3qD0X2W0H2J7Dw+P3tK6sjKr2ZbfjVZ+dXklNXT8hYY3j+itZUzdWgcEwbH4LT3Pc+0koYSBkQM6HLA2t4gMDaMbq/FGF5imcAPRGRHQJnBIlJsLl+F0UpJVUpdD9wBLMHoIvsYeEFEPlBK/RwYD/ybiPgDjvUjYHLAIP/VInLd8Ww8kwWmhdqKJtLX7Gdv+mGcUXbmXDaCCecOwV9WSvnLL1P97nsopYi7/noS7rwDW2Jit527wdPAv3/87xysPchrF73GpIRJ3XZszWlMRS5s/A1k/Qm8Lhh3ieEQMCw1JKera/awvbCmVXS25ddQUtsMgN2qGD84xmzlGMIzMiESyxnqOODxe/jjzj/yu22/46FZD3HduOO+IjukxwXGNOIS4AUMN+XXROQZpdRTwCYRWaOUeha4AvAClcDdIrLbbM38FsOLTICPReRBpVQykA/sBlzmaVaJyKtKKSewGphuHusGEdl/PPv6gsC0UJpXy4Z3cyjcW03cwAjmXTWKEVMT8BQWUf6731Lzj/dRdjvxP/gB/W+/DVu/k5s410J5Uzk3fXgTDZ4GHp3zKJeMuESH+dcY1JdB5iuQ8XtoqoKhc40WzbhLupwELVhKapoNwSmoJutQNd8VVNPg9gEQHWZjitmlNjXZEJ0zIdVAVmkWT6U9xb6qfZyXfB6PzX2MIVFDunSsXiEwvZ2+JDBgjI3kba9gw3s5VJU0Mnh0LGdfM4aBI2Jw5+VR9pvfUPvBP1Hh4fRbupT+/74Ma9zJT5rMq83j4a8eZlflLqYkTOGROY8wNfHUTQTV9HLcDbD1j7BxFVQfgv5jYP5ymHID2E/Ni93nF/aX1bO1pZVTUM3u4jq8fuP9OCTWaYznmK2cyUmxp00k6BpXDS9seYG/7f0bA8MHcW/Cf2DdlcCMC1MYMqZr328tMEHQ1wSmBb/Pz85vi8n4YD9NdR7GzBpA6vdHEZMQjis3l7JVq6j76GMsUVH0W7aMfrfcjDX65Dx//OJnTe4aVm5ZSVlTGRePuJgHZjzA4Kiu5xzRnGH4vLDrfSMUTfE2iBxghqK5DcLjT7k5zR4fO4pqyMo/0r12qLIRAIuCMQOiW8dypg6NZdzA6F4VqFNE+NeBf/Fc5nM013u4Qe6i3/5R1Fe6CI9xsOD6sYyeOaBLx9YCEwR9VWBacDd72frpIbI+O4RfhCnnJTPz4uE4I+0079lL+aqXqPvscyyxsfS/9Vb63fR/sESe3AzqRk8jf8j+A2/ueBOAZROXceukW4mwR3THJWnOBESM5Gffvgi5a8EeCTNvgdR7IG5oj5pW2eBudR7YVmCITlWjBwCn3cLkpFijW22Y0b3WU67SebV5/Hzjzzm49zDzqy9lYMkYxAdJY+OYuCCJkdMSsdq6LoZaYIKgrwtMC/VVLtI/2M/ujcWEhduYfekIJi1Mwmqz0LRjB+UrX6L+q6+wxsfT//bbif/BjVjCTy4UTHF9Mc9veZ6PDnxEYngi98+4n8tHXa7HZzRHU7LdCEWT/a4hPJOuMRwCBk3uacsAo5VwqLKRrBbRya8mu6gWt+kqnRDlaHUgMFymY0PqKu32uXl182t88+V2JhyeT3zDYBxOK+PmDWbSuUn0G9I9IXa0wASBFpijKS+oY8O7OeTvqiImwcm8q0YzakYiSimasrIoW/kSDRs2YE1MIOGOO4m7/josYScXUj2rNIsVmSvYXr6dCf0n8OjsR5kxcEY3XZHmjKE634gOsOVNcNfDqEVGbpoRC086FE134/aartIFR1ylcwNcpUckRDI1+cik0PHd5Cr95baNfPjPjQwuOguHz0lckpNp56Uwds4g7GHd64qtBSYItMC0z6EdFXz7bg6VRQ0MHBHD2deOYfCoWAAaN22i7MWVNGZmYhs0iIS7fkjc1VejHF3/VeYXPx8e+JDnNz9PaWMpF6ZcyAMzHyA5Orm7LklzptBUZcQ7S38Z6g/DoCmG0Ez4Plh776B7bbOH7IKaVieCrPxqSusMR9gWV+lprZNCg3eV9nn8bMvYz5cfbyOsLB6fxUvCJAeLlkxn4IiYkHXPaYEJAi0wHeP3C7s3FpO+Zj+NNW5GTU8k9apRxA2IQERoTEuj7MWVNGVlYU9KIuGeu4m98kqUretf8kZPI2/ueJPXd7yOz+9j6YSl3D75dqIcOp2upg1eF3z3v0aEgIp9EJNkJEKzOY0kaPYI8zPgzxbewb6I9utZHSFtHRXXNJliU0NWfhXbC2qOuEo7bQERCOKZOjSWAdFHPOpqy5vI/rqQrPUHkSYLNc5yoqd6WHrVFcTFnXwonhOhBSYItMCcGI/LR9bnh9jy6SH8Hj+TFiYx69LhhEc5EBEavv6aspUv0ZydjT1lGIk/+hExl16Ksna9SV7SUMLKLSv5YP8H9Hf2597p9/L90d/HehJpejVnKH4/7P3YmLTZVAWeRvA0m59N5l+jkUqgsyhLgPi0I1gdCVPQ+8ztNidYLPj8Qm5ZPVmHqlu713aX1OEzXaWTYpycHR3J8CpBipoQ/ByMz8Z91mHuu/w2Rvcb3c039zi3RgvMidECEzwNNS4y/nmAXd8UYXfamHlxClPOT8ZmtyIi1K9bR9nKl3Dt2YNj1CgSl/+I6IsuQp3EhLntZdtZkbmCrLIsxsWP45HZjzBn8JxuvCpNn8HnaV98vAEi1CpI5rq3ufP7fK4T29IegWIUID4+WzhlTZFsPzyJg+UTcXtjcFvr2T7oW3ITMpjqGcQC5wSGJPZn6ID+DEmIxxoW2bGgmd2I4vEgIli62LWtBSYItMB0noqiejb+PZe87RVE93OS+v2RjJk1EGVRiN9P3aefUfbSS7hzcwkbO5aEe5cT/b3vdbkvWET45OAnPL/5eYoailg0dBE/mfUThsUM6+Yr02i6Ab/PFK62ragAMTruPmNd3E2UVMayvWgiuVXj8IuNyMh9fDXoa7YkZHN5Qw0/qawm3u8/sU0mPo+ioSSSuqII6gutDLrzamKX/6JLl6kFJgi0wHSd/N2VbHg3h/L8egakRDP/mtEkjTUmw4nPR+2HH1G+ahXuvDycEyaQ8KN7iDznnC57nTV7m1m9czWvbH8Fj9/DTeNv4s4pdxLtCH1/s0ZzqnA3e9mbcZjsrwqoKGzA4bQybFYs66Lf48PqvzMidgQ/Tf0pswfOMsahPI2Ip5HCsir2FZaSW1jGodIKSiqqsPtc9G+qZkJZMSMPlxJVWI7yC5ZIB9ETEoi/aRnhFy3tkp1aYIJAC8zJIX5hb0YJae/vp77KxfApCcy/ehTxgwxfe/F6qVnzAeW//S2eggJUWBjh06cTmTqXiLlzCZ80CWXvXBKqssYyXtr6Ev/I+QdxYXEsn76cq8dcrbNnak5rKgrryV5fyJ70EjzNPhKGRjHh3CFsi1vPquyVeP1e7pxyJ8smLsNh7bhbS0Rw7d1HzeefU/HJ56i9Ro7Gw7EDWZ84nrTBE9ndL4XhidH85MJxXDqla5E0tMAEgRaY7sHr9rFtXT6bP87D6/Yz8ZwhzL5sBBExxhdB3G7qv/mWxvQ0GtIzcO3eDYAlIoLw2bOInJtKZOpcws46K+gxm50VO1mRuYLNhzczOm40D89+mPlD5ofsGjtC/ILX48fj8uF1+/C4fXjdfryugGW3D4/r6H2BZY1PHx6XUdbr9iF+GDA8huRxcSSNiyduYIROnnWG4fP4yc0qJfurQopzarDaLIyeNYBJC5Ioj8nn6bSn2Vmxk/lD5vPE3CcYGtN+FAPxemncvIX6dWupW7sOT0EBKEX41KlELV5E9OLFhI0cSW2zh+0FNa2TQm+el8K5Y7oWPV0LTBBogeleGmvdbPrXAbK/LsLmsDDjohSmLh6K3XG095e3spLGjEwa0tNoTEvHfeAAAJbYWCLnzCbCFBzHqFHHfamKCGsPreVXm35FYX0hC5MX8pNZP2FE7IjWMn6fH0/AS771he/2mS/6wBd84PrRQnBsXVMMPMH3gbdgtVuwO6zYwsxPhxWbw4I9zFi2O6z4/UJxTjX1VcagcWSsg6Rx8SSNiyd5XDwxCScXSUHTc9SWN7Hj6yJ2bSiiqc5DTGI4k85NYvz8wXgdLlZlreLt3W/Tz9mPR2c/ykXDLzrme+BvaKD+22+pX7uO+i+/xFdTg3I4iJw3zxCV88/v1tQbbdECEwRaYEJDVUkDG/+ey4Ft5UTGhZF65UjGzR2EajNxTETwe4XGgmLqMrZQt+U7Gr7bgauyFp/FDnEJ2MaOxzpiNJbk4fgjoltf8kYLwVh2u7wUV5dQXleJ1Wcn2hKLU8Lxevz4vZ18vhXmS/7IC79l3RZmxWa3Yg+ztAqBzdzeIhRGnQDhaCsiDusx96EjRITa8iYKdldRuKeKgr3VNNW6AYhJcJI09ojgRMadXEQFTWjx+4VDOyrIXl9IXnYFChg+JYFJC5IYOr4fKPgs7zP+K+O/KGsq47px13HfjPuIccS0HsNbVkbdF19Qv3YdDRs3Im431thYos47j6jFi4g6++yTjhUYLFpggkALTGgp2lfFt3/LoTSvjpgEJzaH1WwBHBGJzj5+Cj82m8IebsfutJsvdeOF77f5OFCfw4HG/Sg7TBk8icmDJhLWUq5FJALEwd5m3Wq39NquKBGhqriRgj1VFO41RMfV6AUgbmAEyWYLJ2lcHOFROjVwb6Cpzs2uDcVkry+krqKZ8BgHE88ZwoRzhhDdz5g4WVhfyDNpz/B14dec1e8sfpr6U6YkTgHAtX8/dZ+vpX7tWpq++w5EsCcnE714EVGLFhMxc8ZJTW7uKlpggkALTOgRv5CzuZS9GSVYrJYjv/jtxq/7Dl/2ZkvA5rAgxQW4t23GvSmD5ow0fDU1ADiGDycidS6RqalEzJnTmiRtT+Uentv0HOnF6YyIHcHDsx7m3ORze/I2hATxC+UF9a2CU7S3Go/LmFDYPynKEJyz4hkyJo6wcO0EcaoQEUpya9j+VSG5W0vxe6XdKMYev4e3drzFy9teRinF8mnLuXHs9Xi276Bu7Vrq167DffAgAM6JE83xlO8RNnZMj/8I0gITBFpgTj/E78e1Zw8Naek0pqXRmJmJv9HI0RE2bhwRc+cQmZpK+MyZfF2zhV9v/jV5tXmcPeRsHpr1EKPjT91s51ONz+enLK/OEJw9VRTn1uDz+FEKEodFk3yW0cIZPCqu24MfakwX4/QSstcXtroYdxTFeGvpVp7a+BQ51TlcMHAhP/afj+3bLdR/8SW+ykqw24mcM8cQlUWLsA8a1ENX1T69QmCUUkuAFzFSJr8qIr9ss38Z8BxQaG5aJSKvmvtWAJcCFuAz4H4REaXUM8DNQLyIRAVzrI7QAnP6Ix4PzTt2GIKTkU7j5i2IywUWC86JE3HOmU36kAZe8HxCtaWJa8dey4+m/Yh456lPYHWq8Xn8lByoaRWcwwdq8fsEi1UxcESMMX4zNp6BI2OwdUM0375KRWE92V+ZLsYuw8V40oKkdqMY17hqeH7z83y67W+cnx/DtSXDCN+6F2lqwhIVRdSCBUR/bzGR55570kn+QkmPC4xSygrsBS4ACoBM4EYR2RlQZhkwS0SWt6k7H0MsFpibvgEeE5EvlVKpQB6wrx2BOeZYx0MLzJmH3+2mKSuLxrR0GtLTadq2DbxesNkoHxnPVwMqyRkZzuJL7uaGyUuxWzs3D+d0xuPyUZxbTeGeagr2VFGWV4uI4dU2eFQsSWPjST4rnsSUaKy9KDNjb+R4LsbtRTEWET755g0y//pbJu5q4KwCUCLYBg0ietEiohYvInL27JOKSn4qCVZgQtkxOwfIEZH9pkHvAFcCO49by0AAJ+AAFGAHDgOISJp5vBCYrDndsTgcRM6ZQ+ScOSRyL/7GRho3b6ExIx1nWjrXbKiAb+px//k5Phy+isHnXsCki240Jn32wGDpqcQeZmXYhP4Mm9AfAFeTl+J9htgU7Kkifc1+0tcY5YaMiWsVnP7JUUGFju8LtOdiPP/q0YyfPxhn1NE/VsTvpzk7m/wP3+Pwx2tIKWkiBWD0cBLuvoSoxYtwTphwRr/LQvmNSgLyA9YLgLntlLtGKbUAo7XzgIjki8hGpdQXQDGGwKwSkV1BnPOYY7UtoJS6E7gTYNgwHc/qTMcSEUHUuecQde45APhqa2nctIl9n79H9IaviX59DXmvr4GIcKJmzyEi1Zz0OW7cSQXqPB0IC7cxfEoCw6ckANBU76Zob3Vrl1pedoVRLsLGkDFxrWM4/QZHntEvxbacyMU40O3c73bTmJZG3dp11K1bh6+sDJ+CshQb7tuXcPb1D+Ac2nfeO6HsIvs34CIRud1cXwrMEZF7A8r0B+pFxKWUugu4TkQWKaVGY4zdXG8W/Qx4VETWB9Stb9NF1u6xjmej7iLr23j8Hv6e/gYb/vV7RuY2Mrc4kpiSOgCscXFEzJnT6qXmGDGiT71UARqqXRTurWoVnNryZgDCo+2t82+SxsUTm9gzeedDTWOtm10bitjxdVGHLsYAvpoa6tevp27tOhrWr8ff2Ig4w9g+2sZXI5rov+hC7j/vcRLCE3rwarqX3jAGMw94UkQuMtcfAxCRZzsobwUqRSRWKfUw4BSRp819PwOaRWRFQPmjBKajYx3PRi0wGjAGXl/e9jLv7H6HQY127vady4x8G00ZmXiLigGwJSYarZu5RivHkdz3sm3WljcdEZzdVTTUGJM+o+LDjhKcwJfv6UbQLsaFha2tlMZNm8DrxZqYgH3BfP6ZVMrrYZkMjk/h8dTHeySEUajpDQJjw+iqWozh2ZUJ/EBEdgSUGSwixebyVRitlFSl1PXAHcASjC6yj4EXROSDgLptWzDtHut4NmqB0QRyoOYAv970a74q+IrkqGQenPkAC9Q4GjMyWp0GfOXlANiTkgLm4MzFPnBAD1t/ahERakqbjPGb3cY8nOZ6DwAxieEkBwhOS0y63syJXIxFBNeuXdR9vpa6deta4+k5Ro0ietEiIhedz7+ce3ghayVN3iZum3Qbt0++Haft9BXb49HjAmMacQnwAoab8msi8oxS6ilgk4isUUo9C1wBeIFK4G4R2W22QH6L4UUmwMci8qB5zBXAD4AhQBGG+/OTHR3rePZpgdG0x4aiDTyX+Rw51TnMHDiTR2Y/woT+ExAR3Lm5hkt0ehoNGZn4WyZ9jhxpzMGZm0rE3DnY4s98N+hAxC9UFjdQsNto4RTtq8bdZEQZiB8caYqN4TjgjOw9nnvHczG2Wfw0ZmYaLZUv1hmtWaUInzGD6EWLiF68CMfw4eyp3MPTaU+zrWwbswfN5onUJxgZO7KnLy2k9AqB6e1ogdF0hNfv5b1977Fq6yqqXdVcOfpK7pt+H4kRRwIIis/XOumzIT2NpsxNRyZ9nnUWkXPnEpE6l4jZs7FGtdube8bi9wvl+XWtcdSKcqrxuv2gICE5qrV1M2R0HI5THGXA5/GTu7WU7PXHuhgnJFpo/Ppr6tauo379evx1dSink8j5843wLOedh62/4YXX6Gnk5W0v89bOt4hxxPDQ7Ie4fOTlZ+R4VFu0wASBFhjNiah11/LKd6/wx11/xG6xc8fkO1g6YWm7XR/i8dCUnU1jejoNaek0bdmCuN1gteKcNLG1dRMxYwaW8L4VDdnn9VN6sLbVYaBkfy0+rx9lUQxIiW4dwxk0KvaY6NvdRUdRjEePseJNMwfp09PB48EaH0/U+ecTvXgRkfPnH/P/+jL/S36R/guKG4q5eszVPDDjAeKccSGxuzeiBSYItMBoguVQ7SH+e/N/s/bQWgZHDubBmQ+2G0Y9EL/LRVPWNqM7LS3dCFbo9aLsdsKnTm11iQ6fMuW0mWDXXXjdPkr2t0QZqKb0YC1+v2CxKQaNiG0VnIEjYloH1rtCRy7GY0dCbO631K9bR3N2NgD2lGFEL1pM9OJFhE+fjrIeK3QlDSX8MuOXrD20llGxo/jpvJ8yc+DMLtt3uqIFJgi0wGg6S0ZxBisyV7Cnag/TEqfxyOxHmJw4Oai6/oYGGrdsoSHNyIPTvHMniKDCw4mYMcNwGpg715h8d4ZP+myLu9lLcc6RsDZl+XUgYHOYUQbGxZM8rh+Jw6KwBBFl4BgX42g7Y0ZCcnkmvvUf48k3psg5p05pFZXj5R/y+r28vfttVm1dhV/8/HDqD7llwi19KhJEIFpggkALjKYr+Pw+3s99n5VbVlLRXMHlIy/nvhn3MSiycwEJfTU1NG7a1Bq407VvHwCWqCjCp0/HGhODCgtDOexYwsJQDgfKYX6GOVAOR5vtAeVatzuwBK6HhaHs9l4/ibS5wUPRvmojD86eKiqLGgBwOM0oA+PMKANDolonOooIxbk1ZAe4GA9M8DOsYTsxae8iVRUou52IealEL1pM1PnnB+X9l12ezVMbn2JX5S7OSTqHx+c+TnJ033NTD0QLTBBogdGcDPXuel7d/iqrd67GoizcOulWlk1aRrita+Mr3vJyGjMyjO607dvxNzYgbg/iciFut/Hp8XSL7cpuP1qIwkwhcgQKlClY9iDLtRHAo4TN3G4xhbF1e5BC11jrbs2BU7CniprSJgCckXaSxsbRLymK/VtLqShswG71k+zLZUDWe0RWH8ISE0PUeQuJXrSYyHPOwRoVXFKuOncdK7es5H/3/C8J4Qk8OudRLky5sE8M4p8ILTBBoAVG0x0U1BUYEXLzPmVgxEDun3E/l468FIvq/laC+P2Ix3NEcNxu/C6XIUTuI0Lkd7sRl9tYd7cp1yJY7jblWo7ndh0jbH7PseW6Bbsdi91+dAvLYcfiCDuuADZZoyn3xVPmiqO0MYpGj50YbzmDcz9l0OFMnIMSiFpsdH1FzJyJsgfflSUifJL3CSsyVlDeVM6NZ93IvdPvJcrRtzwBj4cWmCDQAqPpTjYf3syKzBXskz3A+QAADaxJREFUrNjJ5ITJPDL7EaYNmNbTZoUEETlG6I4SNk87AuhytW4/Rtg8nRHAo9dFBK8tgsixKcQsXkz04sVGLLkutDTy6/J5Jv0Zvi38lvH9xvOf8/6TiQkTQ3AHT2+0wASBFhhNd+MXPx/kfsCLW16krKmMi4dfzI9n/pghUUN62rQzEhEBjwfx+7E4uz5r3uPz8MaON/if7/4Hq7Jy7/R7ueGsG7BZ+pazRbBogQkCLTCaUNHoaeS17Nd4Y8cbANw84WZun3w7EfaInjVMcwybD2/m6Y1Pk1uTywUpF/Do7EcZGDmwp83q1WiBCQItMJpQU1xfzAtbXuDDAx+SGJ7IfTPu44pRV4RkfEbTOaqbq/n15l/zj5x/MCRyCI+nPs6C5AUnrqjRAhMMWmA0p4qs0iyey3yO78q/Y3y/8Twy+xFmDTrh91MTAkSE93Pf59ebfk29u56lE5dy15S7dOuyE2iBCQItMJpTiV/8fHTgI57f/DyHGw9zQcoFPDjzwT4/p+JUsr9mP09vfJpNhzcxLXEaP533U8bGj+1ps047tMAEgRYYTU/Q5G3izR1v8lr2a3j9XpZOWModk+/QbrAhpNnbzCvbX+G17NcIt4Xz4MwHuXrM1bqrsotogQkCLTCanuRww2FWbl3Jmtw19HP2497p93LV6KuwWkIT7LGvsqFwAz9P/zn5dflcNvIyHpr1EP3D+/e0Wac1WmCCQAuMpjeQXZ7NiswVbC3dytj4sTwy+xHmDp7b02ad9pQ3lbMiYwUfHfyIlJgUnkh9gtTBx81BqAkSLTBBoAVG01tomT3+/KbnKWoo4uyksxkVO4owaxhOm5MwaxjhtnDCrGGE2cIIt4YTZgvDaXW2lnFanUdt66stIb/4+euev/Lilhdp9jVzx+Q7uHXyrYRZw3ratDOGYAUmpLOIlFJLgBcxMlq+KiK/bLN/GfAcRkplgFUi8qq5bwVwKWABPgPuFxFRSj0D3AzEt0mZHAa8BcwEKoDrReRg6K5Oo+k+lFIsGb6E84eez+qdq3l799tsObwFl8+FX/xdOqbdYj9KdFqE6hixMpdPVLZlW6uYtSzbnDgsjl4Ro2t35W6e3vg035V/x9xBc3ki9QmGxw7vabP6LCFrwZhpj/cCFwAFQCZwo4jsDCizDJglIsvb1J2PITwtTunfAI+JyJdKqVQgD9jXRmDuAaaIyF1KqRuAq0Tk+uPZqFswmt6OiOD1e2n2NdPsbabZ14zL68Llc9HkbcLlc7VuaynTsq11OeCz2Wcsu7wumnxNrfUCy3UFhTpGhNoVozatrLbbWuu1LHdQ1m45OrZYo6eR32T9hj/t+hOxYbE8NOshLht5Wa8QvTOR3tCCmQPkiMh+06B3gCuBncetZSCAE3AACrADhwFEJM08Xts6VwJPmst/A1YppZT05T5AzWmPUgq71Y7daifaER3y8/nFj9vnPlrA2hGoQLE7alsHAtfgaWhX4Lx+b5fstCrrUWLU4G2gxlXDtWOv5cczfkxsWGw33xlNVwilwCQB+QHrBUB7I5fXKKUWYLR2HhCRfBHZqJT6AijGEJhVIrIr2POJiFcpVQP0B8pP8jo0mj6DRVlau71OxUva5/cdJUjHbaEdR+BEhOvGXXfGBhc9XQmlwLTXNm3bmvgAeFtEXEqpu4A3gUVKqdHAeKBlBtpnSqkFIrL+JM+HUupO4E6AYcOGneASNBpNKLFarERYIvQs+jOUUM4yKgCGBqwnA0WBBUSkQkRaOn1fwRigB7gKSBORehGpBz4CTuRf2Ho+pZQNiAUq2xYSkd+LyCwRmZWYmNjJS9JoNBpNsIRSYDKBMUqpEUopB3ADsCawgFJqcMDqFUBLN9ghYKFSyqaUsgMLA/Z1xBrgFnP5WmCdHn/RaDSaniNkAiMiXmA58AmGOPxFRHYopZ5SSl1hFrtPKbVDKbUNuA9YZm7/G5ALbAe2AdtE5AMw3JeVUgVAhFKqQCn1pFnnD0B/pVQO8CDwH6G6No1Go9GcGD3RUrspazQaTacI1k1ZR3rTaDQaTUjQAqPRaDSakKAFRqPRaDQhQQuMRqPRaEJCnx7kV0qVYcQ16woJ9M4oAdquzqHt6jy91TZtV+c4GbtSROSEEwn7tMCcDEqpTcF4UZxqtF2dQ9vVeXqrbdquznEq7NJdZBqNRqMJCVpgNBqNRhMStMB0nd/3tAEdoO3qHNquztNbbdN2dY6Q26XHYDQajUYTEnQLRqPRaDQhQQtMOyilliil9iilcpRS7QbNVEpdp5TaaQbr/HPA9luUUvvMv1vaq9tDdvmUUlnm35r26obKLqXU8wHn3quUqg7Y12P36wR29eT9GqaU+kIptVUp9Z1S6pKAfY+Z9fYopS7qDXYppYYrpZoC7tfLp9iuFKXUWtOmL5VSyQH7evL5Op5doXy+XlNKlSqlsjvYr5RSK027v1NKzQjY1733S0T0X8AfYMWI5DwSI2XzNmBCmzJjgK1AvLk+wPzsB+w3P+PN5fietstcru+p+9Wm/L3Aa73hfnVkV0/fL4y+8bvN5QnAwYDlbUAYMMI8jrUX2DUcyO7B+/VX4BZzeRGwujc8Xx3ZFcrnyzz2AmBGR/8T4BKMHFsKI89Weqjul27BHMscIEdE9ouIG3gHuLJNmTuA34hIFYCIlJrbLwI+E5FKc99nwJJeYFcoCcauQG4E3jaXe/p+dWRXKAnGLgFizOVYjiTquxJ4R0RcInIAyDGP19N2hZJg7JoArDWXvwjY39PPV0d2hRQxMv8ek2wxgCuBt8QgDYhTRm6ubr9fWmCOJQnID1gvMLcFMhYYq5T6VimVppRa0om6PWEXgFMptcnc/v1usilYuwCjywDjl/e6ztY9xXZBz96vJ4GblJH36EOM1lWwdXvCLoARZtfZV0qpc7vJpmDt2gZcYy5fBUQrpfoHWbcn7ILQPV/B0JHt3X6/tMAci2pnW1tXOxtGd9R5GL98X1VKxQVZtyfsAhgmxqzdHwAvKKVGnUK7WrgB+JuI+LpQt7OcjF3Qs/frRuANEUnG6M5YrZSyBFm3J+wqxrhf0zGS/f1ZKRVD9xCMXQ9hZMDdipH9thDwBlm3J+yC0D1fwdCR7d1+v7TAHEsBMDRgPZljuwIKgPdFxGN2VezBeLEHU7cn7EJEiszP/cCXwPRTaFcLN3B0N1RP36+O7Orp+3Ub8Bfz/BsBJ0bcqJ6+X+3aZXbZVZjbN2OMTYw9VXaJSJGIXG0K3OPmtpogr6kn7Arl8xUMHdne/fcrVANNp+sfRitgP0aXScvg3cQ2ZZYAb5rLCRjNyv4Yg2MHMAbI4s3lfr3ArnggLGD7Po4z4N3ddpnlxgEHMedemdt69H4dx64evV8YA7DLzOXxGF9yBUzk6EH+/XTfIP/J2JXYYgfGoHfhKX7uEwCLufwM8FRveL6OY1fInq+Acw+n40H+Szl6kD8jVPer2y7oTPrDaP7vxfgl9ri57SngCnNZAf8N7AS2AzcE1L0VY/A1B/j33mAXMN9c32Z+3nYq7TLXnwR+2U7dHrtfHdnV0/cLY3D4W/P8WcCFAXUfN+vtAS7uDXZhjDPsMLdvAS4/xXZdi/GS3gu8ivny7unnqyO7TsHz9TZGt6UHo1VyG3AXcJe5XwG/Me3eDswK1f3SM/k1Go1GExL0GIxGo9FoQoIWGI1Go9GEBC0wGo1GowkJWmA0Go1GExK0wGg0Go0mJGiB0WhChFLqSaXUQ73AjoNKqYSetkPT99ACo9FoNJqQoAVGo+kESqlIpdS/lFLblFLZSqnrA1sISqlZSqkvA6pMVUqtM/Nr3GGWGayUWm/mAsluCQ6plPqdGQBxh1Lq/wWc86BS6hdKqY3m/hlKqU+UUrlKqbvMMueZx/y7MvIBvWzGCWtr/01KqQzz3P+jlLKG8n5p+jZaYDSazrEEKBKRqSIyCfj4BOWnYITmmAf8TCk1BCPA4SciMg2YijErHozZ4LPMOguVUlMCjpMvIvOAr4E3MGaJp2LMHG9hDvATYDIwCrg60BCl1HjgeuBs89w+4P904to1mk5h62kDNJrTjO3Ar5RS/wX8U0S+Vqq9ILStvC8iTUCTUuoLDBHIBF5TStmBf4hIi8Bcp5S6E+N7ORgjNMt35r6WrIfbgSgRqQPqlFLNARGzM8QInohS6m3gHOBvAbYsBmYCmabN4cCpyBmk6aNogdFoOoGI7FVKzcSIQ/WsUupTjBDsLb0BzrZVjj2ErFdKLcBo2axWSj2H0TJ5CJgtIlVKqTfaHMtlfvoDllvWW77Hx5yrzbrCCIb62AkuU6PpFnQXmUbTCcwurkYR+SPwK4zUtAcxWgZwJMFUC1cqpZxmoqnzMFoPKUCpiLwC/ME8RgzQANQopQYCF3fBvDlKqRHm2Mv1wDdt9q8FrlVKDTCvpZ9pi0YTEnQLRqPpHJOB55RSfoxotXdjdDX9QSn1f4H0NuUzgH8Bw4CnRaRIKXUL8LBSygPUAzeLyAEzMdUOjDDw33bBto3AL00b1wN/D9wpIjuVUk8An5oi5AF+BOR14VwazQnR0ZQ1mjMApdR5wEMicllP26LRtKC7yDQajUYTEnQLRqPRaDQhQbdgNBqNRhMStMBoNBqNJiRogdFoNBpNSNACo9FoNJqQoAVGo9FoNCFBC4xGo9FoQsL/Bwa34jv1kULuAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch3_1.best_score_, gsearch3_1.best_params_))\n",
    "test_means = gsearch3_1.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch3_1.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch3_1.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch3_1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch3_1.cv_results_).to_csv('my_preds_subsampleh_colsample_bytree_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(colsample_bytree), len(subsample))\n",
    "train_scores = np.array(train_means).reshape(len(colsample_bytree), len(subsample))\n",
    "\n",
    "for i, value in enumerate(colsample_bytree):\n",
    "    pyplot.plot(subsample, -test_scores[i], label= 'test_colsample_bytree:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'subsample' )                                                                                                      \n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig( 'subsample_vs_colsample_bytree1.png' )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 调整 reg_alpha & reg_lambda 参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'reg_alpha': [1.5, 2], 'reg_lambda': [0.5, 1, 2]}"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#reg_alpha = [1e-3, 1e-2, 0.05, 0.1]    #default = 0\n",
    "#reg_lambda = [1e-3, 1e-2, 0.05, 0.1]   #default = 1\n",
    "\n",
    "reg_alpha = [ 1.5, 2]    #default = 0, 测试0.1,1，1.5，2\n",
    "reg_lambda = [0.5, 1, 2]      #default = 1，测试0.1， 0.5， 1，2\n",
    "\n",
    "param_test5_1 = dict(reg_alpha=reg_alpha, reg_lambda=reg_lambda)\n",
    "param_test5_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.58067, std: 0.00348, params: {'reg_alpha': 1.5, 'reg_lambda': 0.5},\n",
       "  mean: -0.58077, std: 0.00333, params: {'reg_alpha': 1.5, 'reg_lambda': 1},\n",
       "  mean: -0.58109, std: 0.00323, params: {'reg_alpha': 1.5, 'reg_lambda': 2},\n",
       "  mean: -0.58037, std: 0.00303, params: {'reg_alpha': 2, 'reg_lambda': 0.5},\n",
       "  mean: -0.58033, std: 0.00295, params: {'reg_alpha': 2, 'reg_lambda': 1},\n",
       "  mean: -0.58078, std: 0.00295, params: {'reg_alpha': 2, 'reg_lambda': 2}],\n",
       " {'reg_alpha': 2, 'reg_lambda': 1},\n",
       " -0.5803315702587188)"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb5_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=231,  #第二轮参数调整得到的n_estimators最优值\n",
    "        max_depth=6,\n",
    "        min_child_weight=4,\n",
    "        gamma=0,\n",
    "        subsample=0.9,\n",
    "        colsample_bytree=0.6,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch5_1 = GridSearchCV(xgb5_1, param_grid = param_test5_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch5_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch5_1.grid_scores_, gsearch5_1.best_params_,     gsearch5_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([229.48057547, 231.03066359, 222.88673916, 228.40723243,\n",
       "        218.10218515, 152.73719978]),\n",
       " 'std_fit_time': array([ 7.94907399, 16.98823651,  2.48241776,  3.95569897, 33.81115817,\n",
       "         5.23866544]),\n",
       " 'mean_score_time': array([1.11266861, 1.03439288, 0.91447783, 0.95591793, 0.76593513,\n",
       "        0.65823212]),\n",
       " 'std_score_time': array([0.05222941, 0.12844592, 0.04594245, 0.09506266, 0.07638405,\n",
       "        0.01180298]),\n",
       " 'param_reg_alpha': masked_array(data=[1.5, 1.5, 1.5, 2, 2, 2],\n",
       "              mask=[False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_reg_lambda': masked_array(data=[0.5, 1, 2, 0.5, 1, 2],\n",
       "              mask=[False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'reg_alpha': 1.5, 'reg_lambda': 0.5},\n",
       "  {'reg_alpha': 1.5, 'reg_lambda': 1},\n",
       "  {'reg_alpha': 1.5, 'reg_lambda': 2},\n",
       "  {'reg_alpha': 2, 'reg_lambda': 0.5},\n",
       "  {'reg_alpha': 2, 'reg_lambda': 1},\n",
       "  {'reg_alpha': 2, 'reg_lambda': 2}],\n",
       " 'split0_test_score': array([-0.57478914, -0.57548033, -0.57590504, -0.57543216, -0.57556793,\n",
       "        -0.57586254]),\n",
       " 'split1_test_score': array([-0.57933041, -0.57888364, -0.57927723, -0.57876166, -0.57913115,\n",
       "        -0.57970072]),\n",
       " 'split2_test_score': array([-0.58109679, -0.58103506, -0.58160977, -0.58078158, -0.5801069 ,\n",
       "        -0.58078145]),\n",
       " 'split3_test_score': array([-0.58352049, -0.5841826 , -0.58364165, -0.5831337 , -0.58322532,\n",
       "        -0.58378968]),\n",
       " 'split4_test_score': array([-0.58463503, -0.58428391, -0.58499496, -0.58371986, -0.58362755,\n",
       "        -0.58378307]),\n",
       " 'mean_test_score': array([-0.58067413, -0.5807729 , -0.58108549, -0.58036559, -0.58033157,\n",
       "        -0.58078331]),\n",
       " 'std_test_score': array([0.00347583, 0.00333473, 0.00323104, 0.00303306, 0.00294688,\n",
       "        0.00294696]),\n",
       " 'rank_test_score': array([3, 4, 6, 2, 1, 5]),\n",
       " 'split0_train_score': array([-0.47253813, -0.4748823 , -0.47742758, -0.47631433, -0.47796808,\n",
       "        -0.47979401]),\n",
       " 'split1_train_score': array([-0.47067687, -0.47261906, -0.47546216, -0.47298078, -0.47397251,\n",
       "        -0.47721303]),\n",
       " 'split2_train_score': array([-0.46939582, -0.47238466, -0.47492412, -0.4716886 , -0.47316958,\n",
       "        -0.47574547]),\n",
       " 'split3_train_score': array([-0.47350703, -0.47427125, -0.47680467, -0.47540369, -0.47687046,\n",
       "        -0.47894839]),\n",
       " 'split4_train_score': array([-0.47152524, -0.47267302, -0.47624682, -0.47482007, -0.47582474,\n",
       "        -0.47834342]),\n",
       " 'mean_train_score': array([-0.47152862, -0.47336606, -0.47617307, -0.47424149, -0.47556108,\n",
       "        -0.47800886]),\n",
       " 'std_train_score': array([0.00142881, 0.00101191, 0.00089927, 0.00167861, 0.00177879,\n",
       "        0.0014098 ])}"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch5_1.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.580332 using {'reg_alpha': 2, 'reg_lambda': 1}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAELCAYAAADkyZC4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xl4VOX1wPHvyUbYl4Q9CfuqLEIIARQERXEDdwVaxaq4lFrtr3WtltJat9a6Vou71gAVrYIKiBW1KCSEfRUhwiQQICRsIWQ/vz/uBcY4kAQymUlyPs8zj3PvvPfNucB48t577vuKqmKMMcZUtZBAB2CMMaZ2sgRjjDHGLyzBGGOM8QtLMMYYY/zCEowxxhi/sARjjDHGLyzBGGOM8QtLMMYYY/zCEowxxhi/CAt0AIEUHR2tHTt2DHQYxhhToyxfvnyvqrYsr12dTjAdO3YkNTU10GEYY0yNIiLbK9LOLpEZY4zxC78mGBEZIyLficgWEbnfx+eTRCRLRFa5r1u8PntSRNaLyEYReU5ExN3/qIiki0humb6Gi8gKESkWkav9eV7GGGPK57cEIyKhwIvARUBvYLyI9PbRdJaq9ndfr7rHDgWGAX2BM4FBwAi3/VwgwUc/HmASkFSV52GMMebU+PMeTAKwRVXTAERkJjAO2FCBYxWIBCIAAcKB3QCqutTt78cHqG5z95eeTtBFRUVkZGSQn59/Ot0YP4mMjCQmJobw8PBAh2KMKYc/E0x7IN1rOwMY7KPdVSIyHNgM3KOq6aq6REQWAZk4CeYFVd3ox1iPB5mRQePGjenYseNPkpgJLFUlOzubjIwMOnXqFOhwjDHl8Oc9GF//dy67utlcoKOq9gU+B94CEJGuQC8gBidRjXKT0OkHJTJZRFJFJDUrK+snn+fn5xMVFWXJJQiJCFFRUTa6NKaG8GeCyQBivbZjgJ3eDVQ1W1UL3M1XgIHu+yuApaqaq6q5wDwgsSqCUtXpqhqvqvEtW/ou47bkErzs78aYmsOfCWYZ0E1EOolIBHA9MMe7gYi09docCxy9DOYBRohImIiE49zgr5ZLZMYYU5sdzC/inSXbSM/J8/vP8luCUdViYAqwACc5/FtV14vINBEZ6za7yy1FXg3chVMFBjAb2AqsBVYDq1V1LhwrX84AGohIhohMdfcPcvdfA/xTRNb769yMMaamWZOxn/vfX8PgR//Lwx+tZ8H6XX7/mX59DkZVP1XV7qraRVUfdfc9oqpz3PcPqOoZqtpPVUeq6iZ3f4mq3qaqvVS1t6r+xqvPe1U1RlVD3P9Odfcvc7cbqmqUqp7hz3OrLb788ksuvfTS025zMps2bWLIkCHUq1ePv/71rydsN2nSJDp16kT//v3p378/q1atOuWfaYyBwwXFzEjxcNnzixn7wjd8tGonY/u146NfDuPms/1fKFOnp4qpCVQVVSUkpOZOutCiRQuee+45Pvzww3LbPvXUU1x9tT0na8zp2LDzIEkp2/lw5U5yC4rp0box08adweVntadJZPWV+FuCOYk/zl3Php0Hq7TP3u2a8IfLTj642rZtGxdddBEjR45kyZIl3H333bz88ssUFBTQpUsX3njjDRo1asSnn37Kb37zG6KjoxkwYABpaWl8/PHHPvtMSUnh7rvv5siRI9SvX5833niDHj16/KjN1KlT2bp1Kzt27CA9PZ17772XW2+9FYDc3Fyuvvpq1q1bx8CBA/nXv/6FiDBt2jTmzp3LkSNHGDp0KP/85z9/ciO+VatWtGrVik8++eQ0/uSMMSdzpLCEj9fsJCnFw0rPfiLCQri0b1smDo5jQFzzgBTI1Nxfi2u57777jhtuuIGFCxfy2muv8fnnn7NixQri4+N5+umnyc/P57bbbmPevHksXrwYXyXX3nr27MnXX3/NypUrmTZtGg8++KDPdmvWrOGTTz5hyZIlTJs2jZ07ncK/lStX8swzz7BhwwbS0tL45ptvAJgyZQrLli1j3bp1HDly5FiCe/nll3n55Zcrfd4PPfQQffv25Z577qGgoKD8A4yp477ffYipc9Yz+C+f87vZazh4pIiHL+1NyoPn8fS1/RnYoUXAqi9tBHMS5Y00/KlDhw4kJiby8ccfs2HDBoYNGwZAYWEhQ4YMYdOmTXTu3PnYA4fjx49n+vTpJ+zvwIED3HjjjXz//feICEVFRT7bjRs3jvr161O/fn1GjhxJSkoKzZo1IyEhgZiYGAD69+/Ptm3bOPvss1m0aBFPPvkkeXl55OTkcMYZZ3DZZZdx++23V/qcH3vsMdq0aUNhYSGTJ0/miSee4JFHHql0P8bUdvlFJcxft4ukZA8p23IIDxUuOrMtEwbHMbhT4BJKWZZgglTDhg0B5x7M6NGjmTFjxo8+X7lyZaX6e/jhhxk5ciT/+c9/2LZtG+eee67PdmX/YR7drlev3rF9oaGhFBcXk5+fz5133klqaiqxsbFMnTr1tB6CbNu27bGfddNNN520IMCYuigtK5cZKR5mL89gX14RHaMa8MBFPbl6YAxRjeqV30E1s0tkQS4xMZFvvvmGLVu2AJCXl8fmzZvp2bMnaWlpbNu2DYBZs2adtJ8DBw7Qvn17AN58880Ttvvoo4/Iz88nOzubL7/8kkGDBp2w7dFkEh0dTW5uLrNnz67Emf1UZmYm4CTVDz/8kDPPPPO0+jOmNigsLuXjNTuZ8MpSRv3tK974ZhtDukTxr5sH88X/ncttI7oEZXIBG8EEvZYtW/Lmm28yfvz4Y/ck/vznP9O9e3f+8Y9/MGbMGKKjo0lI8DXB9HH33nsvN954I08//TSjRo06YbuEhAQuueQSPB4PDz/8MO3atWPz5s0+2zZr1oxbb72VPn360LFjxx8lo6P3X26//XZ27dpFfHw8Bw8eJCQk5Ni9nCZNmnDxxRfz6quv0q5dOyZOnEhWVhaqSv/+/U/pHo4xtYUnO48Zyzy8l5rO3txC2jerz+8u7ME18TG0ahwZ6PAqRFTLTg9Wd8THx2vZFS03btxIr169AhRR5eTm5tKoUSNUlV/+8pd069aNe+6555T7mzp1Ko0aNeK3v/1tFUZZ9WrS35ExlVFcUsrnG/eQlOLh681ZhAic16s1EwbHMbxbS0JDguPeiogsV9X48trZCKYGe+WVV3jrrbcoLCzkrLPO4rbbbgt0SMaYU7Bj/xFmpXiYlZrO7oMFtG0ayd3nd+O6QbG0bVo/0OGdMhvB1OARjC9vvPEGzz777I/2DRs2jBdffDFAEVW9mv53ZAxASany5Xd7SEr2sOi7PShwbveWTBjcgZE9WhIWGry3yG0EU0fddNNN3HTTTYEOwxhzArsP5jNrWTozUzzsPJBPy8b1uPPcrlw3KJbYFg0CHV6VsgRjjDF+Vlqq/G/LXpKSt/P5xj2UlCrndIvm4Ut7c37v1oQH8WjldFiCMcYYP8k6VMB7y9OZmZKOJyePqIYR3HJOJ8YPiqNjdMNAh+d3lmCMMaYKqSpLtmbzboqHz9bvoqhESezcgt9e2IMLz2hNvbDQQIdYbSzBGGNMFcg5XMj7yzOYkeIhbe9hmtYP54YhHRmfEEfXVo0CHV5A1M4Lf6bCqmM9mHfffZe+ffvSt29fhg4dyurVq0+5L2OCiaqS8kMOd89cSeJf/sujn26kRcMInr62H8kPnsfDl/aus8kFbAQT9GrDejCdOnXiq6++onnz5sybN4/JkyeTnJwc6LCMOWUH8or4YGUGSckevt+TS+N6YYxPiGXC4A70aNM40OEFDb8mGBEZAzwLhAKvqurjZT6fBDwF7HB3vaCqr7qfPQlcgjPKWgj8WlVVRB4FbgCaq2ojr77qAW8DA4Fs4DpV3XZaJzDvfti19rS6+Ik2feCix0/apLatBzN06NBj7xMTE8nIyDiVPzljAkpVWZm+n6RkDx+v2Ul+USn9Ypvx5FV9ubRfWxpE2O/rZfnt12IRCQVeBC4CegPjRaS3j6azVLW/+zqaXIYCw4C+wJnAIGCE234u4GvirZuBfaraFfg78ERVnk91q63rwbz22mtcdNFFp/NHY0y1OpRfxDtLt3Pxc4u58h/fMm9tJlcOiOHjX53NR78cxrWDYi25nIA//1QSgC2qmgYgIjOBccCGChyrQCQQAQgQDuwGUNWlbn9ljxkHTHXfzwZeEBHR05mqoJyRhj/VxvVgFi1axGuvvcbixYtP94/HGL9bm3GApJTtfLRqJ3mFJfRu24RHrziTcf3b06ieJZSK8OefUnsg3Ws7Axjso91VIjIc2Azco6rpqrpERBYBmTgJ5gVV3VjRn6eqxSJyAIgC9p7meQREbVsPZs2aNdxyyy3MmzePqKioSsVuTHXJKyxmzipn2eE1GQeIDA9hbL92TBjcgX4xTYNmIa+awp93jn39TZQdTcwFOqpqX+Bz4C0AEekK9AJicBLHKDcJne7PQ0Qmi0iqiKSWd1kpGNSG9WA8Hg9XXnkl77zzDt27dz9pnMYEwsbMgzz84ToGP/pf7v9gLQVFpfxx7BkkP3g+T17dj/6xzSy5nAJ/jmAygFiv7Rhgp3cDVc322nyF4/dNrgCWqmougIjMAxKBryvw8zJEJAxoCuSUbaSq04Hp4Ex2WYnzCYjasB7MtGnTyM7O5s477wQgLCyMspOMGlPd8otK+HhNJknJ21nh2U9EWAiX9nGWHR7YobkllCrgt9mU3f/JbwbOw6kSWwZMUNX1Xm3aqmqm+/4K4D5VTRSR64BbgTE4I5P5wDOqOtfr2NwyVWS/BPqo6u0icj1wpapee7IYa/psyrYejDGVt2XPId5N9vD+8gwO5hfTObohEwbHcdWAGJo3jAh0eDVCwGdTdu+DTAEW4JQpv66q60VkGpCqqnOAu0RkLFCMM9qY5B4+GxgFrMW5zDX/aHJxy5cnAA1EJAOn/Hkq8Brwjohscfu63l/nFixsPRhjKqaguIT563bxbrKHlB9yCA8VxpzZlgkJcSR2bmGjFT+x9WBq8AjGF1sPxpjjfth7mBkpzrLD+/KK6BDVgPEJcVw9MIboIF3HviYI+AimJlPVGvsbTW1fD6Yu/0JkKqawuJSFG3aTlLKdb7ZkExoiXNDbWXZ4WJdoQoJk2eG6wBJMGZGRkWRnZxMVFVVjk0xtpapkZ2cTGRkZ6FBMEErPyWNGiod/p2awN7eA9s3q89sLunNtfCytmti/mUCwBFNGTEwMGRkZ5T4ZbwIjMjLy2AOfxhSXlPLfTc6yw19/n4UAo3q2ZuLgOIZ3b0mojVYCyhJMGeHh4ceejjfGBKed+48wc1k6s5Z52H2wgNZN6nHXqG5cNyiWds3qBzo847IEY4ypEUpKla82O6OVLzbtQYER3Vvyp3FxjOrZirBauuxwTWYJxhgT1PYczGfWsnRmLktnx/4jRDeqxx3nduH6QXHEtmgQ6PDMSViCMcYEndJSZfGWvSQle1i4cTclpcrZXaN56JJejO7dmnAbrdQIlmCMMUFjb24B76U6yw57cvJo0TCCW87uxPUJcXSKbhjo8EwlWYIxxgSUqrIkLZukZA8L1u+iqEQZ3KkF/3dBd8ac2YZ6YaGBDtGcIkswxpiA2He4kPdXZJCU4iEt6zBNIsP4eWJHJgyOpWsrW3a4NrAEY4ypNqpK6vZ9JCV7+GRtJoXFpQzs0Jy/XdOVS/q2JTLcRiu1iSUYY4zfHThSxH/c0crm3bk0rhfG9YNimTA4jp5tmgQ6POMnlmCMMX6hqqxK309Ssoe5a3aSX1RK35imPHFVHy7r187Wsa8D7G/YGFOlcguK+XDlDpKSPWzIPEiDiFCuOCuGiYPjOLN900CHZ6qRJRhjTJVYt+MA7yZ7mLNqB4cLS+jVtgl/vvxMxvVvR+PI8ECHZwLAEowx5pTlFRYzd/VOkpI9rM44QGR4CJf2bcfEwXG2jr2xBGOMqbxNuw6SlOzhPyt2cKigmG6tGjH1st5cMSCGpvVttGIcfk0wIjIGeBZnyeRXVfXxMp9PAp4Cdri7XlDVV93PngQuAUKAhcCvVVVFZCDwJlAf+NRrfz/gZaARsA2YqKoH/Xl+xtQl+UUlfLImk6QUD8u37yMiLIRL+rRlwuA44js0t9GK+Qm/JRgRCQVeBEYDGcAyEZmjqhvKNJ2lqlPKHDsUGAb0dXctBkYAXwIvAZOBpTgJZgwwD3gV+K2qfiUivwB+Bzzsh1Mzpk7ZsieXpGQP76/I4MCRIjpHN+T3l/TiqgExNG8YEejwTBDz5wgmAdiiqmkAIjITGAeUTTC+KBAJRAAChAO7RaQt0ERVl7h9vg1cjpNgegBfu8cvBBZgCcaYU1JQXML8dbtISvaQ/EMO4aHChWe0YcLgOIZ0ttVeTcX4M8G0B9K9tjOAwT7aXSUiw4HNwD2qmq6qS0RkEZCJk2BeUNWNIhLv9uPdZ3v3/TpgLPARcA0QW6VnY0wdsG3vYWakeHhveQY5hwuJbVGf+8b05Jr4GKIb1Qt0eKaG8WeC8fUrjpbZngvMUNUCEbkdeAsYJSJdgV7A0bVxF7pJ6MhJ+vwF8JyIPALMAQp9BiUyGecSG3FxcZU4HWNqp6KSUhZu2E1SsofFW/YSGiKM7tWaCYPjOLtrNCG27LA5Rf5MMBn8eBQRA+z0bqCq2V6brwBPuO+vAJaqai6AiMwDEoF3OJ50ftSnqm4CLnDbd8cpEPgJVZ0OTAeIj48vm/CMqTPSc/KYuczDv1MzyDpUQLumkfzf6O5cOyiW1k0iAx2eqQX8mWCWAd1EpBNOldj1wATvBiLSVlUz3c2xwEb3vQe4VUQewxkJjQCeUdVMETkkIolAMnAD8LzbVytV3SMiIcDvcSrKjDFeiktK+WLTHpJSPHy1OQsBRvVsxYTBcYzo3opQG62YKuS3BKOqxSIyBedmeyjwuqquF5FpQKqqzgHuEpGxQDGQA0xyD58NjALW4lwCm6+qc93P7uB4mfI89wUwXkR+6b7/AHjDX+dmTE2TeeAIM1PSmbUsnV0H82ndpB6/GtWN6wbF0r5Z/UCHZ2opUa27V4ni4+M1NTU10GEY4xclpcrXm7N4N9nDF5t2o8A53VoycXAc5/VsRZgtO2xOkYgsV9X48trZk/zG1DJ7Dubz79R0ZqSks2P/EaIbRXD7iC6MT4gjtkWDQIdn6hBLMMbUAqWlyjdb95KU7GHhht0UlyrDukbx4MW9GN27NRFhNlox1c8SjDE1WHZuAe8tz2BGioft2Xk0bxDOL87uxPiEODpFNwx0eKaOswRjTA2jqixNyyEpxcP8dZkUlSgJHVvwm9HdufCMNrbssAkalmCMqSH25xUye7mz7HBa1mGaRIbxs8QOTEiIo1vrxoEOz5ifsARjTBBTVZZv30dSsoeP12ZSWFzKgLhm/PWaflzSpy31I2y0YoKXJRhjgtDB/CL+s8JZdvi73YdoVC+M6+JjmTA4jl5tmwQ6PGMqxBKMMUFCVVmdcYCk5O3MXZ3JkaIS+rRvyuNX9uGyfu1oWM++rqZmsX+xxgRYbkExH61yRivrdx6kQUQol5/VjgkJHegT0zTQ4RlzyizBGBMg63YcICnFw0crd3C4sISebRrzp8vP5PL+7WgcacsOm5rPEowx1SivsJiPV2fyboqH1en7qRcWwmX92jFhcBxnxTazhbxMrWIJxphq8N2uQyQlb+eDlTs4lF9M11aN+MNlvbnyrBiaNrDRiqmdLMEY4yf5RSV8ujaTpGQPqdv3EREawkV92jBxcAcGdWxuoxVT61mCMaaKbc3KJSnZw/srMtifV0Sn6IY8dHEvrhoYQ4uGEYEOz5hqYwnGmCpQUFzCgvW7SUreztK0HMJChAvPbMPEhDiGdImy0YqpkyzBGHMatmcfJinFw+zUDLIPFxLTvD73junBNQNjadm4XqDDMyagLMEYU0lFJaV8vmE3SSke/vf9XkJDhPN6tmJiYgfO6RpNiC07bAzg5wQjImOAZ3GWTH5VVR8v8/kk4Clgh7vrBVV91f3sSeASIARYCPxaVVVEBnJ8yeRPvfb3B14GInGWYL5TVVP8eX6mbsnYl+csO5yaTtahAto1jeQ3o7tzbXwsbZpGBjo8Y4KO3xKMiIQCLwKjgQxgmYjMUdUNZZrOUtUpZY4dCgwD+rq7FgMjgC+Bl4DJwFKcBDMGmAc8CfxRVeeJyMXu9rlVf2amLikuKWXRd1kkJW/ny81ZAIzs0YqJg+M4t0crQm20YswJ+XMEkwBsUdU0ABGZCYwDyiYYXxRnJBIBCBAO7BaRtkATVV3i9vk2cDlOglHg6CyATYGdVXcqpq7ZdSCfmcs8zFqWTuaBfFo1rsevRnbluoQ42jerH+jwjKkR/Jlg2gPpXtsZwGAf7a4SkeHAZuAeVU1X1SUisgjIxEkwL6jqRhGJd/vx7rO9+/5uYIGI/BXnstrQqj0dU9uVlCpff59FUrKHLzbtoaRUOadbNH+47AzO69WK8FBbdtiYyvBngvF17UDLbM8FZqhqgYjcDrwFjBKRrkAvIMZtt9BNQkdO0ucdOAnqfRG5FngNOP8nQYlMxrnERlxcXCVPydRGew7l816qs+xwxr4jRDWMYPLwzowfFEdcVINAh2dMjeXPBJMBxHptx1DmspWqZnttvgI84b6/AliqqrkAIjIPSATe4XjSKdvnjcCv3ffvAa/6CkpVpwPTAeLj48smPFNHlJYq327NJillO5+t301xqTKkcxT3X9STC3q3ISLMRivGnK5KJRgRCQEaqerBCjRfBnQTkU44VWLXAxPK9NdWVTPdzbHARve9B7hVRB7DGQmNAJ5R1UwROSQiiUAycAPwvHvMTo4XAowCvq/MuZm6ITu3gNnLndHKtuw8mjUI56ZhHRmfEEfnlo0CHZ4xtUq5CUZEkoDbgRJgOdBURJ5W1adOdpyqFovIFGABTpny66q6XkSmAamqOge4S0TG4pQV5wCT3MNn4ySJtTiXwOar6lz3szs4XqY8z30B3Ao8KyJhQD7uZTBjVJXkH3JISvYwf90uCktKGdSxOXef350xZ7YhMtyWHTZ1TMZyaN0bwv1bsCKqJ79KJCKrVLW/iEwEBgL3ActVte9JD6wB4uPjNTU1NdBhGD/Zn1fI+yt2kJS8na1Zh2kcGcZVA2KYMDiO7q0bBzo8Y6rfoV3w+VRYPQNGT4Nhvy73EF9EZLmqxpfXriKXyMJFJBynHPgFVS0SEbt3YYKSqrLCs493kz18siaTguJS+sc246mr+3Jp33bUj7DRiqmDigtg6Uvw9VNQUghn3wPxv/D7j61IgvknsA1YDXwtIh2AityDMaZaqCpb9uTy9fd7eS81nU27DtGoXhjXxMcwIaEDvds1Kb8TY2qrzQtg/v2QkwY9LoYL/gxRXarlR5ebYFT1OeA5r13bRWSk/0Iy5uRUlW3ZeSzZms2StGyWbM1mb24BAGe2b8JjV/ZhbL92NKxnU+2ZOmzv9zD/AdiyEKK6wc/eh64/eXLDrypyk//XwBvAIZzS37OA+4HP/BuaMcdl7HMTiptUMg/kA9CqcT2GdY1iaJcohnSOtudWjMk/CF8/CUtfdm7iX/AoJEyGsOpfi6giv+L9QlWfFZELgZbATTgJxxKM8ZvdB/OPJZRv0/aSnuM8Y9uiYQRDOkeR2MVJKp2jG9paK8YAlJY6N+8/nwqHs+Csn8F5j0CjVgELqSIJ5ui392LgDVVdLfaNNlVsb24BS9OOj1DSsg4D0CQyjMTOUfxiWCeGdImie6vGNh2+MWVlpMK8e2HHcohJgAmzoP2AQEdVoQSzXEQ+AzoBD4hIY6DUv2GZ2u5AXhFLf8g+Nkr5bvchABpGhJLQqQXjBzkrQfZq28RmLDbmRA7tgs//CKuToFEbuOKf0OdaCAmOmSgqkmBuBvoDaaqaJyJROJfJjKmwQ/lFLNuW41zy2prNhsyDqEJkeAiDOrZgbP92DO0SRZ/2TQmzSSWNObniQkh+Cb560ik7HnY3DP8t1Auu57sqUkVWKiIxwAT3ythXXk/VG+PTkcISUrfn8K07Qlm74wAlpUpEaAhnxTXj7vO6M6RLFP1im1IvzJ5NMabCNn/mlh1vhe4XwYWPVlvZcWVVpIrscWAQ8K676y4RGaqqD/g1MlOj5BeVsNKz3y0b3suq9P0UlShhIUK/2GbcMaILQ7tEMaBDc5uaxZhTsXcLLHgAvv/MKTue+D50q96y48qqyCWyi4H+qloKICJvASsBSzB1WFFJKavT9x+7Kb98+z4KiksJEejTvim/OLsTQzpHMahjC3sexZjTkX/QeQJ/6UsQFuk8KJlwW0DKjiurot/8ZjiTUYKzWqSpY4pLSlm/8yBL0px7KKnbcsgrLAGgV9smTBzcgaFdohjUqQVN64cHOFpjaoHSUlgz0yk7zt3tlh3/IaBlx5VVkQTzGLDSXWFSgOHY6KXWKy1VNu46yJKt2SxNyyY5LYdDBcUAdGvViKsHxjCkcxSDO0fRomHw/yZlTI2SsdwtO06FmEEwfga0HxjoqCqtIjf5Z4jIlzj3YQRnNmUr86lljs7ndfSm/NIfstmfVwRAx6gGXNqvHUO6RJHYuQWtGkcGOFpjaqlDu+G/f4RV70Kj1nD5y9D3uqApO66sCl0icxcFm3N0W0Q8gK03XIN5z+f17da9LE3LOTafV/tm9Tm/V2tn+pUuUbRt6t81I4yp84oLIfllp+y4ON+ZRn/474Ku7LiyTvXuqz35VgOl5+SxJC2bpe6zKLsOOvN5tW5Sj7O7OslkaJdoYlvYfF7GVJvvFzplx9lboPsYuPAvQVt2XFmnmmBsPZgaYNeBfJak7T1W6XV0Pq+ohhEkdoliSGcnqdh8XsYEQPZWZ7bj7xdAVFeYOBu6jQ50VFXqhAlGRJ7HdyIRnKoyE2SOzuf17VZnlJK215nPq2n9cAZ3asHNwzoxpEs03Vs3soRiTKAUHHLKjpf8wyk7Hv0nGHx7jSg7rqyTjWBOtpZwhdYZFpExwLNAKPCqqj5e5vNJwFPADnfXC6r6qvvZk8AlOAUFC4Ffq6qKyEDgTaA+8KnX/llAD7efZsB+Ve1fkThrqv15hSxNy3GTyl42784FoFG9MGc+rwSbz8uYoFFaCmtmwecsYWURAAAfBklEQVR/cMqO+090yo4btw50ZH5zwgSjqm+dTsciEgq8CIwGMoBlIjJHVTeUaTpLVaeUOXYoMAzo6+5aDIwAvgReAiYDS3ESzBhgnqpe53X834ADpxN/MDo6n9e3W5xLXmXn87r8rPYM6WzzeRkTdHYsh3n3QcYyp9z4+iSIKXdJ+xrPn49YJwBbVDUNQERmAuOAsgnGFwUigQicS3LhwG4RaQs0UdUlbp9vA5cD844e6C4lcC0wqupOJTDyCotJ3bbv2MON647O5xUWwoC4ZtxzvjufV0wzIsIsoRgTdHL3OGXHK/8FDVvB5S9B3+trbNlxZfkzwbQH0r22M4DBPtpdJSLDgc3APaqarqpL3Ac7M3ESzAuqulFE4t1+vPtsX6a/c4Ddqvq9r6BEZDLOCIi4uOCqtM4vKmGFZx9L3Zvy3vN59Y9txp3ndmFIZ5vPy5igV1wIKf90yo6LjsDQu5yy48gmgY6sWvkzwfi66F+2aGAuMENVC0TkduAtYJSIdAV6ATFuu4VuEjpSgT7HAzNOFJSqTgemA8THxwe0Gq6wuJQ1GfuPPdy43LOPQq/5vG4+uzNDukQR36G5zedlTE3x/edu2fH30O0CuPAxiO4a6KgCoiKzKT/nY/cBIFVVPzrJoRlArNd2DLDTu4GqZnttvgI84b6/AliqqrluDPOAROAdjiedn/QpImHAlUBQzqlQXFLKup0Hjz3cmLptH0eKShCBXm2a8PPE4/N5NYm0+byMqVGyt8KCh2DzPGjRBSb8G7pfGOioAqoivxZHAj2B99ztq4D1wM0iMlJV7z7BccuAbiLSCadK7HpggncDEWnrzhIAMBbY6L73ALeKyGM4I6ERwDOqmikih0QkEUgGbgCe9+ryfGCTqnpfRguY0lJlQ+bBY0sBp/xwfD6v7q0bcW18DEO6RDG4UxTNbT4vY2qmgkPw9V9h6T8gNAJGT4PBd9TKsuPKqkiC6QqMUtViABF5CfgMpzps7YkOUtViEZkCLMApU35dVdeLyDSc0c8cnLVlxgLFOLM1T3IPn41zk34tziWw+V6LnN3B8TLleXjd4MdJYie8POZvqsr3e3KPjVCSf8g5Np9Xp+iGXNa/HUM6R5HYOYqWjesFKkxjTFVQhTX/hoWPQO4u6DcBzv8DNG4T6MiChqie/DaEiHwHJKjqAXe7KZCsqj1FZKWqnlUNcfpFfHy8pqZW6JEen1SVH/YedhfZcmYd3ptbCEBM8/oM6RzF0K5OQrH5vIypRXascMuOU6DdALj4qTpRdnyUiCxX1XJPuCIjmCeBVe6Myken6/+LiDQEPj+tKGuoJVuz+XdqOkvKzOd1TreWx6Zfsfm8jKmFcrO8yo5bwrh/QL/xdabsuLIqMl3/ayLyKc5zLQI8qKpHb6z/zp/BBau0vbl8vTmLxC5RzozDnaPoZPN5GVN7lRRBynT48nEoyoOhU2D4vXWu7LiyKlr7Ogjn+RKAEspUg9U118bHMiEhzhKKMXXBls+dSSn3boauo2HMYxDdLdBR1QgVKVN+HCfBvOvuuktEhqpqnV3VMtymYTGm9stJc8qOv/sUWnS2suNTUJERzMVAf1UtBRCRt4CV2LLJxpjaqCAX/vc3WPKCU3Z8/h8h8Q4Is8rPyqroJbJmOGXEAE39FIsxxgSOKqx9zyk7PpTp3Lw/f6qVHZ+GiiSYx4CV7txgR6vIbPRijKk9dq50yo7Tk6HdWXDtOxA7KNBR1XgVqSKb4ZYoD8JJMPfhrNFijDE1W24WfDENVrwDDaNh3IvOA5NWdlwlKnSJzJ3OZc7RbRHxAME1FbExxlRUSRGkvOKWHR+GIb+EEfdCpN0BqEqnOkWv1ecaY2qmLf91y46/g67nO7Mdt+we6KhqpVNNMAGd5t4YYyot5we37PgTaN4Jxs9yyo7teTa/OWGCEZHn8Z1IBKeqzBhjgl9BLix+Gr59HkLCncqwxDut7LganGwEc7JZIE99hkhjjKkOqrB2Nix82Ck77nu9k1yatA10ZHXGCROMqr5Vdp+ItFHVXf4NyRhjTtPOVW7Z8VJo2x+ufRtiEwIdVZ1T2XswnwID/BGIMcactsN74b/TYMXb0CAKxr4A/Sda2XGAVDbB2N0wY0zwKSmCZa/CosecsuPEO52y4/p2uziQKptgXvFLFMYYc6q2fuGUHWdtgi7nObMdt+wR6KgMlXwiX1X/UZn2IjJGRL4TkS0icr+PzyeJSJaIrHJft3h99qSIrBeRjSLynLhz44vIQBFZ6/Z5bL/72a/cn7deRJ6sTKzGmBom5weYORHeuQKK82H8TPjZ+5ZcgsipPgdTLhEJBV4ERgMZwDIRmaOqG8o0naWqU8ocOxQYBvR1dy0GRgBfAi8Bk4GlOPeExgDzRGQkMA7oq6oFItLKLydmjAmswsPwv6Nlx2Fw3iOQ+EsIjwx0ZKYMvyUYnBUwt6hqGoCIzMRJAGUTjC8KRAIROPd9woHdItIWaKKqS9w+3wYuB+YBdwCPq2oBgKruqdrTMcYElCqsex8+exgO7YS+17llx+0CHZk5AX+WVrQH0r22M9x9ZV0lImtEZLaIxAK4CWQRkOm+FqjqRvf4jBP02R04R0SSReQrEbGpUI2pLTJXwxsXwfs3Q6NW8IvP4MrpllyCnD9HML4qzsrODDAXmOFe0rodeAsYJSJdgV5AjNtuoYgMB46cpM8woDmQiDPz879FpLOq/uhnishknEtsxMXZfJ3GBLXDe+GLP8Hyt5yy48ueg7N+BiGhgY7MVIA/RzAZQKzXdgyw07uBqmYfvaSFU6E20H1/BbBUVXNVNRfnElii22eMVxfefWYAH6gjBSgFossGparTVTVeVeNbtmx5WidojPGTkiJY+jI8PwBW/stZUfJXy2HgjZZcahB/JphlQDcR6SQiEcD1eE35D+DeUzlqLLDRfe8BRohImIiE49zg3+guG3BIRBLd6rEbgI/cYz4ERrn9dse5f7PXP6dmjPGbrYvg5bNh/n3QfiDc8a1TemzPtNQ4frtEpqrFIjIFWACEAq+r6noRmQakquoc4C4RGQsU4yzJPMk9fDZOsliLcwlsvqrOdT+7A3gTqI8zspnn7n8deF1E1gGFwI1lL48ZY4LYvm3ObMebPobmHeH6JOhxsc12XINJXf5/cHx8vKam2rydxgRU4WFY/Hf45jnn8tc5/wdDpljZcRATkeWqGl9eO3/e5DfGmBM7Wna88BE4uAP6XAPn/xGa+io2NTWRJRhjTPXLXOPMduz5Ftr2g6tfh7jEQEdlqpglGGNM9TmcDYv+DMvfhPrN4bJn4ayfW2VYLWUJxhjjfyXFkPoaLHrUWWEy4TY49z4nyZhayxKMMca/0r6C+ffDng3Q+VwY8wS06hnoqEw1sARjjPGPfdvhs4dg41xo1gGuexd6XmJlx3WIJRhjTNUqzHPKjr99DiQERv0ehvzKyo7rIEswxpiqoQrrP4DPHoGDGXDm1TB6mpUd12GWYIwxp2/XWqfsePs30KYPXPUKdBga6KhMgFmCMcacurwc+OLPsPwNiGwGlz4DA26wsmMDWIIxxpyKkmJIfd0tOz4ECZPh3Put7Nj8iCUYY0zl/PC1czlszwboNAIuegJa9Qp0VCYIWYIxxlTMfg989nvY8BE0i4Pr/gU9L7WyY3NClmCMMSdXmAffPAPfPOuUHY/8PQydAuH1Ax2ZCXKWYIwxvqnChg9hwe/dsuOr3LLjmPKPNQZLMMYYX3atc8uOF0PrPnDldOg4LNBRmRrGEowx5ri8HKcyLPV1t+z47zDgRis7NqckxJ+di8gYEflORLaIyP0+Pp8kIlkissp93eL12ZMisl5ENorIcyLOnUQRGSgia90+vfdPFZEdXn1d7M9zM6ZWKSmGlFfg+QGQ+gYMuhV+tRzif2HJxZwyv41gRCQUeBEYDWQAy0RkjqpuKNN0lqpOKXPsUGAY0NfdtRgYAXwJvARMBpYCnwJjgHluu7+r6l+r/myMqcV++J9bdrweOg13Zjtu3TvQUZlawJ+XyBKALaqaBiAiM4FxQNkE44sCkUAEIEA4sFtE2gJNVHWJ2+fbwOUcTzDGmIra74HPHnZu5DeNg2vfhl5jrezYVBl/XiJrD6R7bWe4+8q6SkTWiMhsEYkFcBPIIiDTfS1Q1Y3u8Rkn6XOK29frImKPFBvjS2EefPk4vDAINi+AkQ/BlBToPc6Si6lS/kwwvv6lapntuUBHVe0LfA68BSAiXYFeQAxOAhklIsPL6fMloAvQHycp/c1nUCKTRSRVRFKzsrIqd0bG1GSqsP5DeDEBvnwMelwMU5bBiHvtmRbjF/5MMBlArNd2DLDTu4GqZqtqgbv5CjDQfX8FsFRVc1U1F+cSWKLbp3cR/rE+VXW3qpaoaqnbV4KvoFR1uqrGq2p8y5YtT+sEjakxdq+Hty6D9250qsMmfQLXvAHNYss/1phT5M8EswzoJiKdRCQCuB6Y493Avady1Fhgo/veA4wQkTARCce5wb9RVTOBQyKS6FaP3QB85KOvK4B1/jgpY2qUvBz45Lfw8tmwex1c8jTc9hV0PDvQkZk6wG83+VW1WESmAAuAUOB1VV0vItOAVFWdA9wlImOBYiAHmOQePhsYBazFuQQ2X1Xnup/dAbwJ1McZ2Ry9wf+kiPR3228DbvPXuRkT9EpLnCn0v/gz5B+AQbfAuQ9AgxaBjszUIaJa9rZI3REfH6+pqamBDsOYqrVtsVN2vHsddDzHme249RmBjsrUIiKyXFXjy2tnT/IbU1vsT4eFD8P6/0DTWLjmLasMMwFlCcaYmq7oCHzzHCz+u7N97oMw7C6rDDMBZwnGmJpKFTbOcWY7PuCBM66A0X+yyjATNCzBGFMT7V7v3GfZ9j9ofSZc/jF0OifQURnzI5ZgjKlJ8nKchySXvQaRTeCSv8GASRBqX2UTfOxfpTE1QWkJLH/TLTve78xyPPIhKzs2Qc0SjDHBbts3btnxWqfseMzj0ObMQEdlTLkswRgTrA5kOLMdr//ALTt+E3pfbmXHpsawBGNMsCk6At8+D/97GlAYcT8M+zVENAh0ZMZUiiUYY4KFKmycC5895KzV0vtyuOBP0Cwu0JEZc0oswRgTDHZvgPn3wQ9fQ6sz4Ma5zuqSxtRglmCMCaQj+2DRY7DsVajXGC7+Kwy8ycqOTa1g/4qNCYTSEljxFvz3T07Z8cCbYNTvrezY1CqWYIypbtu/hXn3wq610GGYM9txmz6BjsqYKmcJxpjqciADFj4C696HJjFw9RvO/GFWdmxqKUswxvhbUb5Tdrz4adBSGHEfDLvbyo5NrWcJxhh/UYVNH8OCh2D/dug1Fi74MzTvEOjIjKkWIf7sXETGiMh3IrJFRO738fkkEckSkVXu6xavz54UkfUislFEnhNxriOIyEARWev2eWy/13G/FREVkWh/npsxJ7VnE7xzOcz6GUQ0hBvmwHXvWHIxdYrfRjAiEgq8CIwGMoBlIjJHVTeUaTpLVaeUOXYoMAzo6+5aDIwAvgReAiYDS4FPgTHAPPe4WPfnefxwSsaU78h++PJxSJnulB1f9JQzMaWVHZs6yJ//6hOALaqaBiAiM4FxQNkE44sCkUAEIEA4sFtE2gJNVHWJ2+fbwOW4CQb4O3Av8FEVnocx5SstgRVvwxd/cp5tGTgJRv4eGkYFOjJjAsafCaY9kO61nQEM9tHuKhEZDmwG7lHVdFVdIiKLgEycBPOCqm4UkXi3H+8+2wOIyFhgh6quFqvKMdVp+xK37HgNxA11yo7b9i3/OGNqOX8mGF//l9cy23OBGapaICK3A28Bo0SkK9ALiHHbLXST0BFffYpIA+Ah4IJygxKZjHOJjbg4m+PJnIYDO9yy49nQpD1c/TqccaWVHRvj8udN/gzAe3HwGGCndwNVzVbVAnfzFWCg+/4KYKmq5qpqLs4lsES3zxivLo722QXoBKwWkW3u/hUi0qZsUKo6XVXjVTW+ZcuWp3mKpk4qyoevn4IX4p3JKYffC1OWwZlXWXIxxos/E8wyoJuIdBKRCOB6YI53A/eeylFjgY3uew8wQkTCRCQc5wb/RlXNBA6JSKJbPXYD8JGqrlXVVqraUVU74iSiAaq6y4/nZ+oaVdj4MbyY4Kws2fU8mJICox5yKsWMMT/it0tkqlosIlOABUAo8LqqrheRaUCqqs4B7nLvnRQDOcAk9/DZwChgLc5ltfmqOtf97A7gTaA+zsjm6A1+Y/xnzyaYfz+kLYKWveCGj6DzuYGOypigJqplb4vUHfHx8ZqamhroMEwwO7IfvnoCkv8J9RrByIcg/mYrOzZ1mogsV9X48trZt8QYX0pLYOU7zmzHedlO2fGo30NDe37XmIqyBGNMWZ6lTtlx5mqIGwIXfQBt+wU6KmNqHEswxhx1cCcs/AOs/Tc0bgdXvWaVYcacBkswp+K7ebB2NjRq5VwyadgKGraERi2d/zZsBeGRgY7SVFRRPix5Af73NJQWw/Dfwdn3WGWYMafJEsypyN0DO1Lh8F4ozPXdJqKxV8JxX41a+diOhshm9ltyIKg6vywseAD2bYOel8KFj0LzjoGOzJhawRLMqRh4o/MCKMyDw1nHX7l7frqdvRU8SyAvh59OZgCEhP90BNQw2ndCahAFoeHVerq1UtZ3Ttnx1i+gZU/4+YfQZWSgozKmVrEEc7oiGkBEh4pNw15S7FQkHc6Cw3ucEZCvhLRnk/N5SaHvfuq38JF8vJOT17Zd5vmx/APw5ROQ8k8IbwhjnoBBN1vSNsYPLMFUp9AwaNzaeZVHFQoOQm6WV0LKKrO916l0OrwXCg747ie84YlHQz+6f9TKuVQX4tclggKntBRW/Qs+/6NbdnwjjHrYyo6N8SNLMMFKBCKbOq/oruW3L8qHvKMjor2+E9J+D2SkOu209Kd9hIRBg+gyI6IT3D9q2BLCIqr+vP3Bk+yWHa+C2ET42fvQrn+gozKm1rMEU1uER0LTGOdVntJSOJJz4ntGRxNU9hYnQRX7msQaJ/n5qqDzNWKq17j6CxkOZsLnf4A1s5yy4ytfhT5XW0GFMdXEEkxdFBLiXh6Lhla9Tt5W1amU83V5zjs57dkIuV9B/n7f/YRFVuyeUcNW0KAFhISe+vkVF8CSF+Hrvzplx+f81ik7rtfo1Ps0xlSaJRhzciLO6KNeY2jRufz2xYXOJbifJCSv7UM7ncW5Dmc5CeAnPzPEqZY7WTWd9/2jo88cqcLm+TD/Adj3g1N2fMGfoUWnqv0zMcZUiCUYU7XCIqBJO+dVntJSZ8Tj6/Kcd0LKWHbyZ47qNXESTmg9yNoI0T3g5/+BLqOq9tyMMZViCcYETkiIczmsQQto2aP89uU9c5SX41SHDbrFyo6NCQKWYEzNUZlnjowxAVdLH3owxhgTaJZgjDHG+IUlGGOMMX7h1wQjImNE5DsR2SIi9/v4fJKIZInIKvd1i9dnT4rIehHZKCLPiThPx4nIQBFZ6/bpvf9PIrLG7eczEalAGZMxxhh/8VuCEZFQ4EXgIqA3MF5EevtoOktV+7uvV91jhwLDgL7AmcAgYITb/iVgMtDNfY1x9z+lqn1VtT/wMfCIf87MGGNMRfhzBJMAbFHVNFUtBGYC4yp4rAKRQARQDwgHdotIW6CJqi5RVQXeBi4HUNWDXsc3xOe8+MYYY6qLPxNMeyDdazvD3VfWVe6lrdkiEgugqkuARUCm+1qgqhvd4zNO1KeIPCoi6cBETjCCEZHJIpIqIqlZWVmnfnbGGGNOyp8JxteMgmVHFXOBjqraF/gceAtARLoCvYAYnAQySkSGl9enqj6kqrHAu8AUX0Gp6nRVjVfV+JYtW1bylIwxxlSUPx+0zABivbZjgJ3eDVQ122vzFeAJ9/0VwFJVzQUQkXlAIvCO288J+3QlAZ8AfzhZgMuXL98rItvLPZPqEw3sDXQQJxHs8UHwxxjs8UHwxxjs8UHtj7FCTzv7M8EsA7qJSCdgB3A9MMG7gYi0VdVMd3MssNF97wFuFZHHcEYtI4BnVDVTRA6JSCKQDNwAPO/21U1Vv/fqa1N5AapqUA1hRCRVVeMDHceJBHt8EPwxBnt8EPwxBnt8YDEe5bcEo6rFIjIFWACEAq+r6noRmQakquoc4C4RGQsUAznAJPfw2cAoYC3OJbD5qjrX/ewO4E2gPjDPfQE8LiI9gFJgO3C7v87NGGNM+fw6F5mqfgp8WmbfI17vHwAe8HFcCXDbCfpMxSldLrv/qtON1xhjTNWxJ/mDy/RAB1COYI8Pgj/GYI8Pgj/GYI8PLEYAxHmcxBhjjKlaNoIxxhjjF5Zgqll587O5ba4VkQ3uXGxJwRajiMSJyCIRWek+JHtxNcf3uojsEZF1J/hc3HnqtrjxDQiy+Ca6ca0RkW9FpF91xleRGL3aDRKREhG5urpi8/rZ5cYoIue68w+uF5Gvgik+EWkqInNFZLUb303VHF+s+z3d6P78X/to49/viqraq5peONV0W4HOONPgrAZ6l2nTDVgJNHe3WwVhjNOBO9z3vYFt1RzjcGAAsO4En1+MU10oOM9PJQdZfEO9/n4vqu74KhKj17+FL3AKda4OthiBZsAGIM7dru7vSnnxPQg84b5viVMpG1GN8bUFBrjvGwObfXyX/fpdsRFM9arI/Gy3Ai+q6j4AVd0ThDEq0MR93xTfD7v6jap+jfNlPZFxwNvqWAo0c+exqxblxaeq3x79+wWW8uOHh6tFBf4MAX4FvA9U979BoEIxTgA+UFWP275a46xAfAo0dmd8b+S2La6O2ABUNVNVV7jvD+E8Z1h2ui6/flcswVSviszP1h3oLiLfiMhSERlD9apIjFOBn4lIBs5vt7+qntAqrKLz4AWDmzn+LFfQEJH2ODNqvBzoWE6iO9BcRL4UkeUickOgAyrjBZwpr3biPNP3a1UtDUQgItIROAvnAXVvfv2u+PU5GPMTFZmfLQznMtm5OL/Z/k9EzlTV/X6O7aiKxDgeeFNV/yYiQ4B33BgD8uXxoSLnEHAiMhInwZwd6Fh8eAa4T1VL3CWXglEYMBA4D+fB6yUislRVNwc2rGMuBFbhPDTeBVgoIv/TH8/87nci0ghnJHq3j5/t1++KJZjqVe78bG6bpapaBPwgIt/hJJxl1RNihWK8GXcdHlVdIiKROPMaBeRSig8VOYeAEpG+wKvARfrjOfmCRTww000u0cDFIlKsqh8GNqwfyQD2quph4LCIfA30w7nXEAxuAh5X52bHFhH5AegJpFRXACISjpNc3lXVD3w08et3xS6RVa9j87OJSATO/GxzyrT5EBgJICLROJcB0oIsRg/Ob42ISC+ctXuCae2DOcANboVMInBAj895F3AiEgd8APw8iH7b/hFV7aSqHVW1I87UTXcGWXIB+Ag4R0TCRKQBMJjj8xkGA+/vSWugB9X4XXbv/bwGbFTVp0/QzK/fFRvBVCOt2PxsC4ALRGQDUAL8rjp/w61gjP8HvCIi9+AMpye5v6VVCxGZgXMJMdq9D/QHnEXpUNWXce4LXQxsAfJwfpOsNhWI7xEgCviHO0Io1mqeGLECMQZceTGq6kYRmQ+swZmD8FVVPWnZdXXGB/wJeFNE1uJcirpPVatzhuVhwM+BtSKyyt33IBDnFaNfvyv2JL8xxhi/sEtkxhhj/MISjDHGGL+wBGOMMcYvLMEYY4zxC0swxhhj/MISjDHGGL+wBGNMDeBOS//x6bYxpjpZgjGmCrhPQtv3yRgv9oUw5hSJSEd3Mad/ACuAn4vIEhFZISLvuZMMIiIXi8gmEVnsLu50wlGGiCSIswjZSve/PXy0mSoi74jIFyLyvYjc6vVxIxGZ7f68d93pQhCRR0RkmYisE5HpR/cb40+WYIw5PT2At4HROJOAnq+qA4BU4DfuRKD/xJnU8mychadOZhMwXFXPwplS5i8naNcXuAQYAjwi8v/t3TFok0EYxvH/AypShIJuTlJUHFwKOjm5F0QQxE0HHeqgk3OhgyCCiwri5NRVEUFwqEODpaPQqVAp6iA6KFRKh/I43BVKIakxuWbw+S1JLseb+5a83H0f76vjdXwSuEtpBDdBKRcC8Nj2edtnKZWHp/q+0og+pRZZxGDWbC9KmqL8qXfq5uAQ8IFSPXfV9qc6fw641SPeOPBC0ilKnbeDXea9sr0BbEiapzSK+wks2f4CUOtPnQAWgIuS7gFjwFFgGXj9b5cc8XeSYCIG87u+Cnhn+9rOLyVN9hlvFpi3fbk2iXrfZd7uIoLbnzd3jG0BB+ou6ilwzvZnSTOUCtgRTeWILGI4FoELkk4CSBqTdJpy5DVRkwXA1T3ijANf6/vrPeZdknRY0jFKRd9e/YK2k8mPel/oyh5riBiKJJiIIbD9nZIQ5iR9pCScM/UYaxp4K2kB+Ab86hHqAXBfUofSLqGbJeBN/Z1Z212bRNVuqM8pbXtfsn/N6+I/l3L9EY1JOmJ7vT659QRYsf1ogHgzwLrth8NaY0QL2cFEtHez3nBfphyBPRvxeiL2RXYwESMg6QZwZ9dwx/btUawnooUkmIiIaCJHZBER0UQSTERENJEEExERTSTBREREE0kwERHRxB/Ezd7VIcXdjgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch5_1.best_score_, gsearch5_1.best_params_))\n",
    "test_means = gsearch5_1.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch5_1.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch5_1.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch5_1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch5_1.cv_results_).to_csv('my_preds_reg_alpha_reg_lambda_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(reg_alpha), len(reg_lambda))\n",
    "train_scores = np.array(train_means).reshape(len(reg_alpha), len(reg_lambda))\n",
    "\n",
    "#log_reg_alpha = [0,0,0,0]\n",
    "#for index in range(len(reg_alpha)):\n",
    "#   log_reg_alpha[index] = math.log10(reg_alpha[index])\n",
    "    \n",
    "for i, value in enumerate(reg_alpha):\n",
    "    pyplot.plot(reg_lambda, -test_scores[i], label= 'reg_alpha:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'reg_alpha' )                                                                                                      \n",
    "pyplot.ylabel( '-Log Loss' )\n",
    "pyplot.savefig( 'reg_alpha_vs_reg_lambda1.png' )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " ## 调用模型进行测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_test samples: (6317, 227)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:2026: FutureWarning: From version 0.21, test_size will always complement train_size unless both are specified.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "# 用train_test_split 分割训练数据和测试数据\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_train, y_train, train_size = 0.8,random_state = 0)\n",
    "\n",
    "print(\"X_test samples:\" ,X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\preprocessing\\label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n"
     ]
    }
   ],
   "source": [
    "xgb_final = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=231,  #第二轮参数调整得到的n_estimators最优值\n",
    "        max_depth=6,\n",
    "        min_child_weight=4,\n",
    "        gamma=0,\n",
    "        subsample=0.9,\n",
    "        colsample_bytree=0.6,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        reg_alpha=2, \n",
    "        reg_lambda=1, \n",
    "        seed=3)\n",
    "xgb_final.fit(X_train, y_train)\n",
    "y_pred = xgb_final.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.7435491530789932\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "accuracy = accuracy_score(y_test, y_pred)\n",
    "print('Accuracy:', accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 生成测试结果文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>...</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "      <th>interest_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.5</td>\n",
       "      <td>3</td>\n",
       "      <td>3000</td>\n",
       "      <td>1200.0</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>-1.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>5465</td>\n",
       "      <td>2732.5</td>\n",
       "      <td>1821.666667</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2850</td>\n",
       "      <td>1425.0</td>\n",
       "      <td>1425.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3275</td>\n",
       "      <td>1637.5</td>\n",
       "      <td>1637.500000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>18</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>4</td>\n",
       "      <td>3350</td>\n",
       "      <td>1675.0</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>28</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 228 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathrooms  bedrooms  price  price_bathrooms  price_bedrooms  room_diff  \\\n",
       "0        1.5         3   3000           1200.0      750.000000       -1.5   \n",
       "1        1.0         2   5465           2732.5     1821.666667       -1.0   \n",
       "2        1.0         1   2850           1425.0     1425.000000        0.0   \n",
       "3        1.0         1   3275           1637.5     1637.500000        0.0   \n",
       "4        1.0         4   3350           1675.0      670.000000       -3.0   \n",
       "\n",
       "   room_num  Year  Month  Day       ...        walk  walls  war  washer  \\\n",
       "0       4.5  2016      6   24       ...           0      0    0       0   \n",
       "1       3.0  2016      6   12       ...           0      0    0       0   \n",
       "2       2.0  2016      4   17       ...           0      0    0       0   \n",
       "3       2.0  2016      4   18       ...           0      0    0       0   \n",
       "4       5.0  2016      4   28       ...           0      0    1       0   \n",
       "\n",
       "   water  wheelchair  wifi  windows  work  interest_level  \n",
       "0      0           0     0        0     0               1  \n",
       "1      0           0     0        0     0               2  \n",
       "2      0           0     0        0     0               0  \n",
       "3      0           0     0        0     0               2  \n",
       "4      0           0     0        0     0               2  \n",
       "\n",
       "[5 rows x 228 columns]"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dpath = './data/'\n",
    "pd.set_option('display.max_info_columns', 500)\n",
    "# 读取训练集\n",
    "train = pd.read_csv(dpath + \"RentListingInquries_FE_train.csv\") \n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>...</th>\n",
       "      <th>virtual</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2950</td>\n",
       "      <td>1475.000000</td>\n",
       "      <td>1475.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>11</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2850</td>\n",
       "      <td>1425.000000</td>\n",
       "      <td>950.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3758</td>\n",
       "      <td>1879.000000</td>\n",
       "      <td>1879.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>3300</td>\n",
       "      <td>1650.000000</td>\n",
       "      <td>1100.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>11</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.0</td>\n",
       "      <td>2</td>\n",
       "      <td>4900</td>\n",
       "      <td>1633.333333</td>\n",
       "      <td>1633.333333</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 227 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathrooms  bedrooms  price  price_bathrooms  price_bedrooms  room_diff  \\\n",
       "0        1.0         1   2950      1475.000000     1475.000000        0.0   \n",
       "1        1.0         2   2850      1425.000000      950.000000       -1.0   \n",
       "2        1.0         1   3758      1879.000000     1879.000000        0.0   \n",
       "3        1.0         2   3300      1650.000000     1100.000000       -1.0   \n",
       "4        2.0         2   4900      1633.333333     1633.333333        0.0   \n",
       "\n",
       "   room_num  Year  Month  Day  ...   virtual  walk  walls  war  washer  water  \\\n",
       "0       2.0  2016      6   11  ...         0     0      0    0       0      0   \n",
       "1       3.0  2016      6   24  ...         0     0      0    1       0      0   \n",
       "2       2.0  2016      6    3  ...         0     0      0    0       0      0   \n",
       "3       3.0  2016      6   11  ...         0     0      0    0       0      0   \n",
       "4       4.0  2016      4   12  ...         0     0      0    1       0      0   \n",
       "\n",
       "   wheelchair  wifi  windows  work  \n",
       "0           0     0        0     0  \n",
       "1           0     0        0     0  \n",
       "2           0     0        0     0  \n",
       "3           1     0        0     0  \n",
       "4           0     0        0     0  \n",
       "\n",
       "[5 rows x 227 columns]"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 读取测试集\n",
    "pred = pd.read_csv(dpath + \"RentListingInquries_FE_test.csv\")\n",
    "pred.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 74659 entries, 0 to 74658\n",
      "Data columns (total 227 columns):\n",
      "bathrooms                 74659 non-null float64\n",
      "bedrooms                  74659 non-null int64\n",
      "price                     74659 non-null int64\n",
      "price_bathrooms           74659 non-null float64\n",
      "price_bedrooms            74659 non-null float64\n",
      "room_diff                 74659 non-null float64\n",
      "room_num                  74659 non-null float64\n",
      "Year                      74659 non-null int64\n",
      "Month                     74659 non-null int64\n",
      "Day                       74659 non-null int64\n",
      "Wday                      74659 non-null int64\n",
      "Yday                      74659 non-null int64\n",
      "hour                      74659 non-null int64\n",
      "top_10_manager            74659 non-null int64\n",
      "top_25_manager            74659 non-null int64\n",
      "top_5_manager             74659 non-null int64\n",
      "top_50_manager            74659 non-null int64\n",
      "top_1_manager             74659 non-null int64\n",
      "top_2_manager             74659 non-null int64\n",
      "top_15_manager            74659 non-null int64\n",
      "top_20_manager            74659 non-null int64\n",
      "top_30_manager            74659 non-null int64\n",
      "cenroid                   74659 non-null int64\n",
      "distance                  74659 non-null float64\n",
      "display_address_pred_0    0 non-null float64\n",
      "display_address_pred_1    0 non-null float64\n",
      "display_address_pred_2    0 non-null float64\n",
      "1br                       74659 non-null int64\n",
      "24                        74659 non-null int64\n",
      "2br                       74659 non-null int64\n",
      "3br                       74659 non-null int64\n",
      "ac                        74659 non-null int64\n",
      "access                    74659 non-null int64\n",
      "actual                    74659 non-null int64\n",
      "air                       74659 non-null int64\n",
      "allowed                   74659 non-null int64\n",
      "appliances                74659 non-null int64\n",
      "approval                  74659 non-null int64\n",
      "apt                       74659 non-null int64\n",
      "areas                     74659 non-null int64\n",
      "attended                  74659 non-null int64\n",
      "available                 74659 non-null int64\n",
      "backyard                  74659 non-null int64\n",
      "balcony                   74659 non-null int64\n",
      "basement                  74659 non-null int64\n",
      "bath                      74659 non-null int64\n",
      "bathroom                  74659 non-null int64\n",
      "bedford                   74659 non-null int64\n",
      "bedroom                   74659 non-null int64\n",
      "bedrooms.1                74659 non-null int64\n",
      "bike                      74659 non-null int64\n",
      "billiards                 74659 non-null int64\n",
      "bldg                      74659 non-null int64\n",
      "blks                      74659 non-null int64\n",
      "brand                     74659 non-null int64\n",
      "brick                     74659 non-null int64\n",
      "brownstone                74659 non-null int64\n",
      "building                  74659 non-null int64\n",
      "business                  74659 non-null int64\n",
      "cable                     74659 non-null int64\n",
      "cats                      74659 non-null int64\n",
      "ceiling                   74659 non-null int64\n",
      "ceilings                  74659 non-null int64\n",
      "center                    74659 non-null int64\n",
      "central                   74659 non-null int64\n",
      "chef                      74659 non-null int64\n",
      "children                  74659 non-null int64\n",
      "childrens                 74659 non-null int64\n",
      "clean                     74659 non-null int64\n",
      "cleaning                  74659 non-null int64\n",
      "close                     74659 non-null int64\n",
      "closet                    74659 non-null int64\n",
      "closets                   74659 non-null int64\n",
      "club                      74659 non-null int64\n",
      "common                    74659 non-null int64\n",
      "concierge                 74659 non-null int64\n",
      "conditioning              74659 non-null int64\n",
      "construction              74659 non-null int64\n",
      "courtyard                 74659 non-null int64\n",
      "deck                      74659 non-null int64\n",
      "decorative                74659 non-null int64\n",
      "details                   74659 non-null int64\n",
      "dining                    74659 non-null int64\n",
      "dishwasher                74659 non-null int64\n",
      "dogs                      74659 non-null int64\n",
      "doorman                   74659 non-null int64\n",
      "dry                       74659 non-null int64\n",
      "dryer                     74659 non-null int64\n",
      "duplex                    74659 non-null int64\n",
      "eat                       74659 non-null int64\n",
      "elev                      74659 non-null int64\n",
      "elevator                  74659 non-null int64\n",
      "entertainment             74659 non-null int64\n",
      "exclusive                 74659 non-null int64\n",
      "exposed                   74659 non-null int64\n",
      "facilities                74659 non-null int64\n",
      "fee                       74659 non-null int64\n",
      "fireplace                 74659 non-null int64\n",
      "fitness                   74659 non-null int64\n",
      "flex                      74659 non-null int64\n",
      "floor                     74659 non-null int64\n",
      "floors                    74659 non-null int64\n",
      "free                      74659 non-null int64\n",
      "friendly                  74659 non-null int64\n",
      "ft                        74659 non-null int64\n",
      "fully                     74659 non-null int64\n",
      "furnished                 74659 non-null int64\n",
      "garage                    74659 non-null int64\n",
      "garden                    74659 non-null int64\n",
      "granite                   74659 non-null int64\n",
      "green                     74659 non-null int64\n",
      "gut                       74659 non-null int64\n",
      "gym                       74659 non-null int64\n",
      "hardwood                  74659 non-null int64\n",
      "health                    74659 non-null int64\n",
      "hi                        74659 non-null int64\n",
      "high                      74659 non-null int64\n",
      "highrise                  74659 non-null int64\n",
      "home                      74659 non-null int64\n",
      "hour.1                    74659 non-null int64\n",
      "housekeeping              74659 non-null int64\n",
      "huge                      74659 non-null int64\n",
      "included                  74659 non-null int64\n",
      "indoor                    74659 non-null int64\n",
      "intercom                  74659 non-null int64\n",
      "internet                  74659 non-null int64\n",
      "kitchen                   74659 non-null int64\n",
      "large                     74659 non-null int64\n",
      "laundry                   74659 non-null int64\n",
      "level                     74659 non-null int64\n",
      "light                     74659 non-null int64\n",
      "live                      74659 non-null int64\n",
      "living                    74659 non-null int64\n",
      "lndry                     74659 non-null int64\n",
      "lobby                     74659 non-null int64\n",
      "loft                      74659 non-null int64\n",
      "lot                       74659 non-null int64\n",
      "lounge                    74659 non-null int64\n",
      "lowrise                   74659 non-null int64\n",
      "luxury                    74659 non-null int64\n",
      "marble                    74659 non-null int64\n",
      "massive                   74659 non-null int64\n",
      "microwave                 74659 non-null int64\n",
      "midrise                   74659 non-null int64\n",
      "modern                    74659 non-null int64\n",
      "month                     74659 non-null int64\n",
      "multi                     74659 non-null int64\n",
      "natural                   74659 non-null int64\n",
      "new                       74659 non-null int64\n",
      "newly                     74659 non-null int64\n",
      "ok                        74659 non-null int64\n",
      "outdoor                   74659 non-null int64\n",
      "oversized                 74659 non-null int64\n",
      "park                      74659 non-null int64\n",
      "parking                   74659 non-null int64\n",
      "patio                     74659 non-null int64\n",
      "pet                       74659 non-null int64\n",
      "pets                      74659 non-null int64\n",
      "photos                    74659 non-null int64\n",
      "playroom                  74659 non-null int64\n",
      "pool                      74659 non-null int64\n",
      "post                      74659 non-null int64\n",
      "pre                       74659 non-null int64\n",
      "prewar                    74659 non-null int64\n",
      "private                   74659 non-null int64\n",
      "publicoutdoor             74659 non-null int64\n",
      "queen                     74659 non-null int64\n",
      "ramp                      74659 non-null int64\n",
      "reduced                   74659 non-null int64\n",
      "renovated                 74659 non-null int64\n",
      "rent                      74659 non-null int64\n",
      "residents                 74659 non-null int64\n",
      "rise                      74659 non-null int64\n",
      "roof                      74659 non-null int64\n",
      "roofdeck                  74659 non-null int64\n",
      "rooftop                   74659 non-null int64\n",
      "room                      74659 non-null int64\n",
      "sauna                     74659 non-null int64\n",
      "service                   74659 non-null int64\n",
      "services                  74659 non-null int64\n",
      "share                     74659 non-null int64\n",
      "shared                    74659 non-null int64\n",
      "shares                    74659 non-null int64\n",
      "short                     74659 non-null int64\n",
      "simplex                   74659 non-null int64\n",
      "site                      74659 non-null int64\n",
      "sized                     74659 non-null int64\n",
      "skylight                  74659 non-null int64\n",
      "space                     74659 non-null int64\n",
      "spacious                  74659 non-null int64\n",
      "speed                     74659 non-null int64\n",
      "sprawling                 74659 non-null int64\n",
      "ss                        74659 non-null int64\n",
      "stainless                 74659 non-null int64\n",
      "steel                     74659 non-null int64\n",
      "steps                     74659 non-null int64\n",
      "storage                   74659 non-null int64\n",
      "studio                    74659 non-null int64\n",
      "sublet                    74659 non-null int64\n",
      "subway                    74659 non-null int64\n",
      "super                     74659 non-null int64\n",
      "superintendent            74659 non-null int64\n",
      "swimming                  74659 non-null int64\n",
      "tenant                    74659 non-null int64\n",
      "term                      74659 non-null int64\n",
      "terrace                   74659 non-null int64\n",
      "time                      74659 non-null int64\n",
      "tons                      74659 non-null int64\n",
      "train                     74659 non-null int64\n",
      "true                      74659 non-null int64\n",
      "tv                        74659 non-null int64\n",
      "unit                      74659 non-null int64\n",
      "utilities                 74659 non-null int64\n",
      "valet                     74659 non-null int64\n",
      "video                     74659 non-null int64\n",
      "view                      74659 non-null int64\n",
      "views                     74659 non-null int64\n",
      "virtual                   74659 non-null int64\n",
      "walk                      74659 non-null int64\n",
      "walls                     74659 non-null int64\n",
      "war                       74659 non-null int64\n",
      "washer                    74659 non-null int64\n",
      "water                     74659 non-null int64\n",
      "wheelchair                74659 non-null int64\n",
      "wifi                      74659 non-null int64\n",
      "windows                   74659 non-null int64\n",
      "work                      74659 non-null int64\n",
      "dtypes: float64(9), int64(218)\n",
      "memory usage: 129.3 MB\n"
     ]
    }
   ],
   "source": [
    "pred.info(verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 74659 entries, 0 to 74658\n",
      "Data columns (total 227 columns):\n",
      "bathrooms                 74659 non-null float64\n",
      "bedrooms                  74659 non-null int64\n",
      "price                     74659 non-null int64\n",
      "price_bathrooms           74659 non-null float64\n",
      "price_bedrooms            74659 non-null float64\n",
      "room_diff                 74659 non-null float64\n",
      "room_num                  74659 non-null float64\n",
      "Year                      74659 non-null int64\n",
      "Month                     74659 non-null int64\n",
      "Day                       74659 non-null int64\n",
      "Wday                      74659 non-null int64\n",
      "Yday                      74659 non-null int64\n",
      "hour                      74659 non-null int64\n",
      "top_10_manager            74659 non-null int64\n",
      "top_25_manager            74659 non-null int64\n",
      "top_5_manager             74659 non-null int64\n",
      "top_50_manager            74659 non-null int64\n",
      "top_1_manager             74659 non-null int64\n",
      "top_2_manager             74659 non-null int64\n",
      "top_15_manager            74659 non-null int64\n",
      "top_20_manager            74659 non-null int64\n",
      "top_30_manager            74659 non-null int64\n",
      "cenroid                   74659 non-null int64\n",
      "distance                  74659 non-null float64\n",
      "display_address_pred_0    74659 non-null int64\n",
      "display_address_pred_1    74659 non-null int64\n",
      "display_address_pred_2    74659 non-null int64\n",
      "1br                       74659 non-null int64\n",
      "24                        74659 non-null int64\n",
      "2br                       74659 non-null int64\n",
      "3br                       74659 non-null int64\n",
      "ac                        74659 non-null int64\n",
      "access                    74659 non-null int64\n",
      "actual                    74659 non-null int64\n",
      "air                       74659 non-null int64\n",
      "allowed                   74659 non-null int64\n",
      "appliances                74659 non-null int64\n",
      "approval                  74659 non-null int64\n",
      "apt                       74659 non-null int64\n",
      "areas                     74659 non-null int64\n",
      "attended                  74659 non-null int64\n",
      "available                 74659 non-null int64\n",
      "backyard                  74659 non-null int64\n",
      "balcony                   74659 non-null int64\n",
      "basement                  74659 non-null int64\n",
      "bath                      74659 non-null int64\n",
      "bathroom                  74659 non-null int64\n",
      "bedford                   74659 non-null int64\n",
      "bedroom                   74659 non-null int64\n",
      "bedrooms.1                74659 non-null int64\n",
      "bike                      74659 non-null int64\n",
      "billiards                 74659 non-null int64\n",
      "bldg                      74659 non-null int64\n",
      "blks                      74659 non-null int64\n",
      "brand                     74659 non-null int64\n",
      "brick                     74659 non-null int64\n",
      "brownstone                74659 non-null int64\n",
      "building                  74659 non-null int64\n",
      "business                  74659 non-null int64\n",
      "cable                     74659 non-null int64\n",
      "cats                      74659 non-null int64\n",
      "ceiling                   74659 non-null int64\n",
      "ceilings                  74659 non-null int64\n",
      "center                    74659 non-null int64\n",
      "central                   74659 non-null int64\n",
      "chef                      74659 non-null int64\n",
      "children                  74659 non-null int64\n",
      "childrens                 74659 non-null int64\n",
      "clean                     74659 non-null int64\n",
      "cleaning                  74659 non-null int64\n",
      "close                     74659 non-null int64\n",
      "closet                    74659 non-null int64\n",
      "closets                   74659 non-null int64\n",
      "club                      74659 non-null int64\n",
      "common                    74659 non-null int64\n",
      "concierge                 74659 non-null int64\n",
      "conditioning              74659 non-null int64\n",
      "construction              74659 non-null int64\n",
      "courtyard                 74659 non-null int64\n",
      "deck                      74659 non-null int64\n",
      "decorative                74659 non-null int64\n",
      "details                   74659 non-null int64\n",
      "dining                    74659 non-null int64\n",
      "dishwasher                74659 non-null int64\n",
      "dogs                      74659 non-null int64\n",
      "doorman                   74659 non-null int64\n",
      "dry                       74659 non-null int64\n",
      "dryer                     74659 non-null int64\n",
      "duplex                    74659 non-null int64\n",
      "eat                       74659 non-null int64\n",
      "elev                      74659 non-null int64\n",
      "elevator                  74659 non-null int64\n",
      "entertainment             74659 non-null int64\n",
      "exclusive                 74659 non-null int64\n",
      "exposed                   74659 non-null int64\n",
      "facilities                74659 non-null int64\n",
      "fee                       74659 non-null int64\n",
      "fireplace                 74659 non-null int64\n",
      "fitness                   74659 non-null int64\n",
      "flex                      74659 non-null int64\n",
      "floor                     74659 non-null int64\n",
      "floors                    74659 non-null int64\n",
      "free                      74659 non-null int64\n",
      "friendly                  74659 non-null int64\n",
      "ft                        74659 non-null int64\n",
      "fully                     74659 non-null int64\n",
      "furnished                 74659 non-null int64\n",
      "garage                    74659 non-null int64\n",
      "garden                    74659 non-null int64\n",
      "granite                   74659 non-null int64\n",
      "green                     74659 non-null int64\n",
      "gut                       74659 non-null int64\n",
      "gym                       74659 non-null int64\n",
      "hardwood                  74659 non-null int64\n",
      "health                    74659 non-null int64\n",
      "hi                        74659 non-null int64\n",
      "high                      74659 non-null int64\n",
      "highrise                  74659 non-null int64\n",
      "home                      74659 non-null int64\n",
      "hour.1                    74659 non-null int64\n",
      "housekeeping              74659 non-null int64\n",
      "huge                      74659 non-null int64\n",
      "included                  74659 non-null int64\n",
      "indoor                    74659 non-null int64\n",
      "intercom                  74659 non-null int64\n",
      "internet                  74659 non-null int64\n",
      "kitchen                   74659 non-null int64\n",
      "large                     74659 non-null int64\n",
      "laundry                   74659 non-null int64\n",
      "level                     74659 non-null int64\n",
      "light                     74659 non-null int64\n",
      "live                      74659 non-null int64\n",
      "living                    74659 non-null int64\n",
      "lndry                     74659 non-null int64\n",
      "lobby                     74659 non-null int64\n",
      "loft                      74659 non-null int64\n",
      "lot                       74659 non-null int64\n",
      "lounge                    74659 non-null int64\n",
      "lowrise                   74659 non-null int64\n",
      "luxury                    74659 non-null int64\n",
      "marble                    74659 non-null int64\n",
      "massive                   74659 non-null int64\n",
      "microwave                 74659 non-null int64\n",
      "midrise                   74659 non-null int64\n",
      "modern                    74659 non-null int64\n",
      "month                     74659 non-null int64\n",
      "multi                     74659 non-null int64\n",
      "natural                   74659 non-null int64\n",
      "new                       74659 non-null int64\n",
      "newly                     74659 non-null int64\n",
      "ok                        74659 non-null int64\n",
      "outdoor                   74659 non-null int64\n",
      "oversized                 74659 non-null int64\n",
      "park                      74659 non-null int64\n",
      "parking                   74659 non-null int64\n",
      "patio                     74659 non-null int64\n",
      "pet                       74659 non-null int64\n",
      "pets                      74659 non-null int64\n",
      "photos                    74659 non-null int64\n",
      "playroom                  74659 non-null int64\n",
      "pool                      74659 non-null int64\n",
      "post                      74659 non-null int64\n",
      "pre                       74659 non-null int64\n",
      "prewar                    74659 non-null int64\n",
      "private                   74659 non-null int64\n",
      "publicoutdoor             74659 non-null int64\n",
      "queen                     74659 non-null int64\n",
      "ramp                      74659 non-null int64\n",
      "reduced                   74659 non-null int64\n",
      "renovated                 74659 non-null int64\n",
      "rent                      74659 non-null int64\n",
      "residents                 74659 non-null int64\n",
      "rise                      74659 non-null int64\n",
      "roof                      74659 non-null int64\n",
      "roofdeck                  74659 non-null int64\n",
      "rooftop                   74659 non-null int64\n",
      "room                      74659 non-null int64\n",
      "sauna                     74659 non-null int64\n",
      "service                   74659 non-null int64\n",
      "services                  74659 non-null int64\n",
      "share                     74659 non-null int64\n",
      "shared                    74659 non-null int64\n",
      "shares                    74659 non-null int64\n",
      "short                     74659 non-null int64\n",
      "simplex                   74659 non-null int64\n",
      "site                      74659 non-null int64\n",
      "sized                     74659 non-null int64\n",
      "skylight                  74659 non-null int64\n",
      "space                     74659 non-null int64\n",
      "spacious                  74659 non-null int64\n",
      "speed                     74659 non-null int64\n",
      "sprawling                 74659 non-null int64\n",
      "ss                        74659 non-null int64\n",
      "stainless                 74659 non-null int64\n",
      "steel                     74659 non-null int64\n",
      "steps                     74659 non-null int64\n",
      "storage                   74659 non-null int64\n",
      "studio                    74659 non-null int64\n",
      "sublet                    74659 non-null int64\n",
      "subway                    74659 non-null int64\n",
      "super                     74659 non-null int64\n",
      "superintendent            74659 non-null int64\n",
      "swimming                  74659 non-null int64\n",
      "tenant                    74659 non-null int64\n",
      "term                      74659 non-null int64\n",
      "terrace                   74659 non-null int64\n",
      "time                      74659 non-null int64\n",
      "tons                      74659 non-null int64\n",
      "train                     74659 non-null int64\n",
      "true                      74659 non-null int64\n",
      "tv                        74659 non-null int64\n",
      "unit                      74659 non-null int64\n",
      "utilities                 74659 non-null int64\n",
      "valet                     74659 non-null int64\n",
      "video                     74659 non-null int64\n",
      "view                      74659 non-null int64\n",
      "views                     74659 non-null int64\n",
      "virtual                   74659 non-null int64\n",
      "walk                      74659 non-null int64\n",
      "walls                     74659 non-null int64\n",
      "war                       74659 non-null int64\n",
      "washer                    74659 non-null int64\n",
      "water                     74659 non-null int64\n",
      "wheelchair                74659 non-null int64\n",
      "wifi                      74659 non-null int64\n",
      "windows                   74659 non-null int64\n",
      "work                      74659 non-null int64\n",
      "dtypes: float64(6), int64(221)\n",
      "memory usage: 129.3 MB\n"
     ]
    }
   ],
   "source": [
    "pred[\"display_address_pred_0\"] = 0\n",
    "pred[\"display_address_pred_1\"] = 0\n",
    "pred[\"display_address_pred_2\"] = 0\n",
    "pred.info(verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = train['interest_level']\n",
    "\n",
    "remove_train = train.drop([\"interest_level\"], axis=1)\n",
    "X_train = np.array(remove_train)\n",
    "\n",
    "X_pred = np.array(pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "reg_alpha = 2    #default = 0, 测试0.1,1，1.5，2\n",
    "reg_lambda = 1      #default = 1，测试0.1， 0.5， 1，2\n",
    "\n",
    "param_test_final = dict(reg_alpha=reg_alpha, reg_lambda=reg_lambda)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\preprocessing\\label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n"
     ]
    }
   ],
   "source": [
    "xgb_final = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=231,  #第二轮参数调整得到的n_estimators最优值\n",
    "        max_depth=6,\n",
    "        min_child_weight=4,\n",
    "        gamma=0,\n",
    "        subsample=0.9,\n",
    "        colsample_bytree=0.6,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        reg_alpha=2, \n",
    "        reg_lambda=1, \n",
    "        seed=3)\n",
    "xgb_final.fit(X_train, y_train)\n",
    "y_pred = xgb_final.predict(X_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "#生成提交测试结果\n",
    "df = pd.DataFrame({'interest_level':y_pred})\n",
    "df.to_csv('submission.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
